import sys, os, time, re, sha, urlparse, HTMLParser, pickle, cPickle, logging, tempfile
from twisted.python import util
from twisted.internet import reactor, defer, task
from twisted.web import client
waitFor = defer.waitForDeferred
from axiom.store import Store
from axiom.item import Item
from axiom.attributes import bytes, ieee754_double

from utils import *
from scrapercore import *
from sitescrapers import *

FIELDS = ["torrentname"]#, "torrenturl", "age", "category",
#          "size", "files", "seeders", "leechers", "downloads",
#          "torrentindurl", "numfiles", "added", "indurl", "magnet",
#          "torrenturlmaybeind", "infohash", "source", "time"]
#
# DB Schema
#
class Torrent(Item):
    schemaVersion = 2
    typeName = "torrent"
    torrentid = bytes(allowNone=False)
    torrentname = bytes()
    torrenturl = bytes()
    age = bytes()
    category = bytes()
    size = bytes()
    files = bytes()
    seeders = bytes()
    leechers = bytes()
    downloads = bytes()
    torrentindurl = bytes()
    numfiles = bytes()
    added = bytes()
    magnet = bytes()
    torrenturlmaybeind = bytes()
    infohash = bytes()
    source = bytes()
    time = ieee754_double()

#for x in FIELDS:
#    setattr(Torrent, x, bytes())

def main():
    #
    # Set up logging
    #

    log = logging.getLogger("Scraper")
    log.setLevel(logging.DEBUG)

    logging.basicConfig(level=logging.DEBUG,
                        filename="scraperdebug.log",
                        filemode="a",
                        format="%(asctime)s %(name)-12s %(levelname)-8s %(message)s",
                        datefmt="%m-%d %H:%M:%S")
    fileLogHandler = logging.FileHandler("scraper.log")
    fileLogHandler.setLevel(logging.INFO)
    fileLogHandler.setFormatter(logging.Formatter(
        "%(asctime)s %(name)-12s %(levelname)-8s %(message)s", "%m-%d %H:%M:%S"))
    logging.getLogger('').addHandler(fileLogHandler)
    stderrHandler = logging.StreamHandler()
    stderrHandler.setLevel(logging.INFO)
    stderrHandler.setFormatter(logging.Formatter(
        "%(asctime)s %(name)-12s %(levelname)-8s %(message)s", "%m-%d %H:%M:%S"))
    logging.getLogger('').addHandler(stderrHandler)



    #
    # Load DB
    #
    torrentStore = Store("torrents.axiom")

    def addTorrent(ti):
        t = torrentStore.findFirst(Torrent, Torrent.torrentid == ti.getID())
        if t is None:
            t = Torrent(store=torrentStore, torrentid=ti.getID())
        for x in ti:
            if x in [s[0] for s in Torrent.getSchema()]:
                try:
                    setattr(t, x, ti[x])
                except Exception, e:
                    print "Exception setting" + x + "to" + str(ti[x]) + " : " + str(e)

    throttler = ClientThrottler(log, 30, 3, 15)         # XXX


#    SCRAPERS = [BitTorrentScraper]
    SCRAPERS = [IsoHuntLatestScraper, IsoHuntIndexedScraper, BitTorrentScraper,
                MininovaScraper, PirateBayLatestScraper, PirateBayTopScraper,
                BitenovaScraper, TorrentSpyScraper, TorrentzScraper,
                BTJunkieLatestScraper, BTJunkieTopScraper,
                TorrentReactorLatestScraper, TorrentReactorTopScraper,
                MeganovaLatestScraper, TorrentPortalScraper]

    scrapers = [x(logging.getLogger(x.__name__),
                  throttler, addTorrent)
                for x in SCRAPERS]

    def getIndexes(scrapers):
        log.info("Fetching indexes")
        for x in scrapers:
            x.getIndex()


    getIndexesTask = task.LoopingCall(lambda: getIndexes(scrapers))
    getIndexesTask.start(3*60*60)

    reactor.run()

if __name__ == "__main__":
    main()