import sys, os, time, re, sha, urlparse, HTMLParser, pickle, cPickle, logging, tempfile
from twisted.python import util
from twisted.internet import reactor, defer, task
from twisted.web import client
waitFor = defer.waitForDeferred
from utils import *

subscraped = set()

class HTMLStripper(HTMLParser.HTMLParser):
    def __init__(self):
        self.reset()
        self.fed = []
    def handle_data(self, d):
        self.fed.append(d)
    def get_fed_data(self):
        return ''.join(self.fed)

def stripHTML(text):
    x = HTMLStripper()
    x.feed(text)
    return x.get_fed_data()


class TorrentInfo(dict):
    def __init__(self, ident):
        dict.__init__(self)
        self["id"] = ident

    def getID(self):
        return self["id"]
        
    def __cmp__(self, other):
        if not isinstance(other, TorrentInfo):
            return -1
        elif "id" not in self:
            return -1
        elif "id" not in other:
            return -1
        else:
            return cmp(self.getID(), other.getID())

    def __hash__(self):
        return hash(self.getID())

class ClientThrottler:
    def __init__(self, log, delayTime, retryCount, retryTime):
        self.delayTime = delayTime
        self.retryCount = retryCount
        self.retryTime = retryTime
        self.history = {}
        self.queue = []
        self.checkTask = task.LoopingCall(self.checkQueue)
        self.checkTask.start(2)
        self.log = log


    def delayedGetPage(self, page, ignoreDelay=False):
        dfd = defer.Deferred()
        self.queue.append((page, ignoreDelay, dfd))
        return dfd

    def checkQueue(self):
        for x in self.queue:
            page, ignoreDelay, dfd = x
            host = urlparse.urlparse(page)[1]
            if (ignoreDelay or host not in self.history
                or (self.history[host] + self.delayTime < time.time())):
                self.history[host] = time.time()
                self.log.debug("Delayed fetch: %s", page)
                self.getPageRetries(page, self.retryCount,
                                    self.retryTime).addCallback(
                    dfd.callback).addErrback(
                    dfd.errback)
                self.queue.remove(x)
                return

    def getPageRetries(self, url, num, delay):
        dfd = defer.Deferred()
        def retry(err, triesLeft):
            if triesLeft <= 0:
                dfd.errback(Exception("Failed to fetch %s after %d tries - %s"
                                      % (url, num, err)))
            else:
                self.log.debug("Failed to fetch %s (%d tries left) - %s",
                          url, triesLeft, err)
                client.getPage(url).addCallback(dfd.callback).addErrback(
                    lambda x : reactor.callLater(delay,
                                                 retry, x, triesLeft-1))
            
        client.getPage(url).addCallback(dfd.callback).addErrback(
            lambda x: retry(x, num-1))
        return dfd
                

class Scraper:
    def __init__(self, log, throttler, addTorrent, url, regex):
        self.addTorrent = addTorrent
        self.url = url
        self.regex = re.compile(regex, re.M)
        self.transforms = {}
        self.taggers = []
        self.log = log
        self.throttler = throttler
        def tagTime(ti):
            ti["time"] = time.time()
        self.taggers.append(tagTime)
        
    def getIndex(self):
        dfd = defer.Deferred()
        self.log.debug("Fetching %s", self.url)
        self.throttler.delayedGetPage(self.url, True).addCallback(
            self.parseIndexPage).addErrback(
            self.log.error).addCallback(dfd.callback)
        return dfd

    def parseIndexPage(self, page):
        dicts = [x.groupdict() for x in
                 self.regex.finditer(page)]

        if len(dicts) > 0:
            self.log.debug("Fetched index page, found %d entries", len(dicts))
            return self.parseDicts(dicts)
        else:
            z = self.dumpPageToFile(page, "index", self.url)
            self.log.warning("Failed to parse index page; saving to %s",
                        z)
            return None

    def parseDicts(self, dicts):
        dfds = []
        for x in dicts:
            dfd = self.makeTorrentInfoFromDict(x)
            dfd.addCallback(self.addTorrentInfoToList)
            dfds.append(dfd)
        return defer.DeferredList(dfds)

    def makeTorrentInfoFromDict(self, d):
        ti = TorrentInfo(self.getIDFromDict(d))
        rdfd = defer.Deferred()
        dfds = []
        for k, v in d.iteritems():
            if k in self.transforms:
                dfds.append(defer.maybeDeferred(
                    self.transforms[k], ti, k, v))
            else:
                ti[k] = v
        for x in self.taggers:
            dfds.append(defer.maybeDeferred(x, ti))
        dfdlist = defer.DeferredList(dfds)
        dfdlist.addCallback(lambda x: rdfd.callback(ti))
        dfdlist.addErrback(rdfd.errback)
        return rdfd
    
    def getIDFromDict(self, d):
        try:
            return d["torrentname"]
        except KeyError:
            return None

    def addTorrentInfoToList(self, ti):
        self.log.debug("Adding: %s", ti.getID())
        self.addTorrent(ti)

    def dumpPageToFile(self, page, prefix, url):
        urlprefix = prefix + "-" + urlparse.urlparse(url)[1] + "-"
        (fd, name) = tempfile.mkstemp(prefix=urlprefix,
                                        suffix=".html",
                                        dir="./failed/")
        os.write(fd, page)
        os.close(fd)
        return name

    def transformRelativeURL(self, ti, k, v):
        ti[k] = urlparse.urljoin(self.url, v)

    def transformStripHTML(self, ti, k, v):
        ti[k] = stripHTML(v)

    def transformStripNBSP(self, ti, k, v):
        ti[k] = v.replace("&nbsp;", " ")


class Subscraper(Scraper):
    def __init__(self, log, throttler, ti, regex):
        Scraper.__init__(self, log, throttler, None, None, regex)
        self.ti = ti

    def getIndex(self):
        raise RuntimeException("Can't getIndex on a subscraper")

    def subscrape(self, url, atMostOnce=True):
        dfd = defer.Deferred()
        self.url = url
        global subscraped
        if atMostOnce and self.url in subscraped:
            self.log.debug("Skipping subscrape of %s", self.url)
            dfd.callback(None)
        else:
            self.log.debug("Subscraping %s", self.url)
            self.throttler.delayedGetPage(self.url).addCallback(
                self.parseSubscrapePage).addErrback(
                self.log.error).addCallback(
                self.registerAsSubscraped).addCallback(
                dfd.callback)
        return dfd

    def addTorrentInfoToList(self, ti):
        self.log.debug("Adding subscraped info to %s", self.ti.getID())
        if ti["id"] != None:
            oldID = ti["id"]
        else:
           oldID = self.ti.getID()
        self.ti.update(ti)
        self.ti["id"] = oldID
        
    def parseSubscrapePage(self, page):
        dicts = [x.groupdict() for x in
                 self.regex.finditer(page)]
        if len(dicts) > 0:
            self.log.debug("Fetched subscrape page, found %d entries",
                           len(dicts))
            return self.parseDicts(dicts)
        else:
            z = self.dumpPageToFile(page, "subscrape", self.url)
            self.log.warning("Failed to parse subscrape page; saving to %s",
                        z)
            return None

    def registerAsSubscraped(self, x):
        global subscraped
        subscraped.add(self.url)
        return x