#! /usr/local/bin/python
### SimSensor.py, by Ilya A. Zemskov(zemskov@univer.omsk.su)

"""
 About this model
**********************************************************************
 Модель работы сенсоров поисковой системы
 - Единица времени равняется 10msc (милисекунд)
 - Размер страницы измеряется в байтах
**********************************************************************

"""

__version__ = '$ Revision: 0.9.0 $ Date: 2003/06/01 16:09:00 $'

from __future__ import generators
from time import localtime, strftime
from random import Random,expovariate,uniform,randint,random
from SimPy.Simulation import *
from SimPy.Monitor import Monitor
import MySQLdb
from ConfigParser import *
from os import getcwd
import sys

#--[ Simulation variables ]------------------------------------
BadChanges = [1,2,3]
WasntChange = [6]
#---------------------------------------------------------------
comini = "..\\ini\\common.ini"
dbini = "..\\ini\\db.ini"

#--[ Special variables ]----------------------------------------
try:
    taskdir = sys.argv[1]
except IndexError, msg:
    sys.exit("Error: missing parameter - name of task directory")

try:
    taskinifile = sys.argv[2]
except IndexError, msg:
    sys.exit("Error: missing parameter - name of ini file")

comcfg = ConfigParser()
comcfg.read(comini)
wdir = comcfg.get('Options','WorkDirectory')

dbcfg = ConfigParser()
dbcfg.read(dbini)

cfg = ConfigParser()
cfg.read(wdir+taskdir+'\\ini\\'+taskinifile)

testcfg = ConfigParser()
testcfg.read(wdir+taskdir+'\\ini\\'+cfg.get('testOptions','CommonIni'))

flog = file(testcfg.get('logOptions','LogDirectory')+
            testcfg.get('logOptions','LogFileName'),
            "a+")

dbconn = MySQLdb.Connect(dbcfg.get('dbOptions','Host'),
                         dbcfg.get('dbOptions','Login'),
                         dbcfg.get('dbOptions','Password'),
                         cfg.get('dbOptions','MainDBName'))

statdbconn = MySQLdb.Connect(dbcfg.get('dbOptions','Host'),
                             dbcfg.get('dbOptions','Login'),
                             dbcfg.get('dbOptions','Password'),
                             cfg.get('dbOptions','StatDBName'))

TRACETOFILE = testcfg.getint('logOptions','LogTraceToFile')
RANDOMSEED = cfg.getint('simOptions','RandomSeed')

simvarsdefault = {\
    'SumSize'      :0,\
    'MinWaitCrawl' :testcfg.getint('simOptions','ModelTime'),\
    'MaxWaitCrawl' :0,\
    'AvrWaitCrawl' :Monitor('AvrWaitCrawl'),\
    'MaxProcessing':0,\
    'NowProcessing':0}

statvarstime = {\
    'needfresh'    :10000,\
    'freshness'    :10000,\
    'sumsize'      :10000,\
    'nowprocessing':10000}

#---------------------------------------------------------------
def putfile(st):
    if TRACETOFILE:
        flog.write(st+"\n")
        flog.flush()
    else:
        print st

#--[ Simulation classes ]----------------------------------------
class Page:
    """ 
    """
    def __init__(self, conn, pid, traceout=0):
        self.conn = conn
        self.pid = pid
        curs = self.conn.cursor()
        curs.execute("select * from %s where pid=%d"%
                     (dbcfg.get('dbOptions','PagesTableName'),
                      self.pid))
        self.page = curs.fetchone()
        self.iter = self.page[1]
        self.lastch  = self.page[2]
        self.size = self.page[3]
        self.nsize = self.page[4]
        self.changed = self.page[5]
        self.crawled = self.page[6]
        self.wtime = self.page[7]
        self.traceout = traceout
        curs.close()

    def update(self):
        curs = self.conn.cursor()
        curs.execute("update %s set lastch=%d, nsize=%d, changed=%d, crawled=%d, wtime=%d where pid=%d"%
                     (dbcfg.get('dbOptions','PagesTableName'),
                      self.lastch,
                      self.nsize,
                      self.changed,
                      self.crawled,
                      self.wtime,
                      self.pid))
        curs.close()

class Changes(Process):
    """ 
    """
    def __init__(self, conn, iter, traceout=0):
        Process.__init__(self)
        self.name = 'ChangeProcess'
        self.conn = conn
        self.iter = iter
        self.traceout = traceout
        
    def life(self):
        curs = self.conn.cursor()
        t = now()
        while 1:
            curs.execute("select etime from %s where etime>%d && iter=%d order by etime asc limit 1"%
                         (dbcfg.get('dbOptions','ChangesTableName'),
                          t,
                          self.iter))
            tt = curs.fetchone()
            if tt==None:
                ChTime = testcfg.getint('simOptions','ModelTime')+1
                print "finish ",now()
            else:
                ChTime = tt[0]-t
            yield hold,self,ChTime
            curs.execute("select * from %s where etime=%d && iter=%d"%
                         (dbcfg.get('dbOptions','ChangesTableName'),
                          now(),
                          self.iter))
            ev = curs.fetchone()
            k=0
            while ev:
                k+=1
                p = Page(self.conn, ev[2])
                p.nsize=ev[6]
                p.lastch=ev[4]
                if not ((ev[4]==ev[5] and ev[4] in [1, 2, 3, 6]) or\
                        (ev[4]==6 and ev[5] in [4, 5])):
                    p.changed = 1
                    if p.crawled:
                        p.crawled = 0
                        p.wtime = now()
                p.update()
                ev = curs.fetchone()
            if k>1: self.trace("%d pages processed"%k)
            t = now()
        curs.close()
        
    def trace(self,message):
        if self.traceout:
            putfile("at %7.4f %s %s"%\
                  (now() , self.name, message))

class Queries(Process):
    """ 
    """
    def __init__(self, conn, iter, traceout=0):
        Process.__init__(self)
        self.name = 'QueryProcess'
        self.conn = conn
        self.iter = iter
        self.traceout = traceout
        
    def life(self):
        curs = self.conn.cursor()
        t = now()
        while 1:
            curs.execute("select qtime from %s where qtime>%d && iter=%d order by qtime asc limit 1"%
                         (dbcfg.get('dbOptions','QueriesTableName'),
                          t,
                          self.iter))
            tt = curs.fetchone()
            if tt==None:
                QryTime = testcfg.getint('simOptions','ModelTime')+1
                print "finish ",now()
            else:
                QryTime = tt[0]-t
            yield hold,self,QryTime
            curs.execute("select * from %s where qtime=%d && iter=%d"%
                         (dbcfg.get('dbOptions','QueriesTableName'),
                          now(),
                          self.iter))
            qu = curs.fetchone()
            k=0
            while qu:
                k+=1
                p = Page(self.conn, qu[2])
                if p.changed:
                    cr = Crawl(p)
                    activate(cr,cr.life(),now())
                qu = curs.fetchone()
            if k>1: self.trace("%d pages queried"%k)
            t = now()
        curs.close()
        
    def trace(self,message):
        if self.traceout:
            putfile("at %7.4f %s %s"%\
                  (now() , self.name, message))

class Crawl(Process):
    """ 
    """
    def __init__(self, page, traceout=0):
        Process.__init__(self)
        self.page = page
        self.name = 'CrawlProcess'
        self.traceout = traceout

    def life(self):
        global grv
        self.trace("Crawl started")
        AlertTime = grv.randint(testcfg.getint('sensorOptions','MinSendAlertTime'),
                                testcfg.getint('sensorOptions','MaxSendAlertTime'))
        yield hold,self,AlertTime
        if self.page.lastch not in BadChanges:
            size = self.page.nsize
            LdTime = grv.randint(testcfg.getint('pageOptions','MinPageLoadTime'),
                                 testcfg.getint('pageOptions','MaxPageLoadTime'))
            self.trace("begin loading page %s"%self.page.pid)
        else:
            size = 0
            LdTime = 0
            self.trace("WARNING!!! page type of change is %s"%self.page.lastch)
            self.trace("loadtime for Page%s is %d"%(self.page.pid,LdTime))
        self.page.crawled = 1
        self.page.changed = 0

        waitcrawl = now() - self.page.wtime
        simvars['MaxWaitCrawl'] = max(simvars['MaxWaitCrawl'],waitcrawl)
        simvars['MinWaitCrawl'] = min(simvars['MinWaitCrawl'],waitcrawl)
        self.page.wtime = testcfg.getint('simOptions','ModelTime')+1
        self.page.update()
        simvars['AvrWaitCrawl'].tally(waitcrawl)
        
        simvars['NowProcessing'] += 1
        simvars['MaxProcessing'] = max(simvars['MaxProcessing'],simvars['NowProcessing'])
        yield hold,self,LdTime
        simvars['NowProcessing'] -= 1

        simvars['SumSize'] += size
        self.cancel()
        
    def trace(self,message):
        if self.traceout:
            putfile("at %7.4f %s %s"%\
                  (now() , self.name, message))

#--[ Statistics classes ]----------------------------------------
class StatVar(Process):
    """ 
    """
    def __init__(self, conn, varname, looktime, iter):
        Process.__init__(self)
        self.var = varname
        self.iter = iter
        self.name = varname+`iter`
        self.looktime = looktime
        self.stime = now()
        self.value = None
        self.conn = conn
        curs = self.conn.cursor()
        try:
            curs.execute("drop table %s"%self.name)
        except MySQLdb.Error,msg:
            print "MySQL error:",msg
        curs.execute("create table %s(%s)"%\
                     (self.name,\
                     dbcfg.get('dbOptions','StatTableFieldsSQL')))
        curs.close()

    def life(self):
        td = 0
        while 1:
            self.setvar()
            self.flushstat()
            ttime = testcfg.getint('simOptions','ModelTime')-now()
            if ttime<self.looktime:
                yield hold,self, ttime + td
                td = self.looktime
            else:
                yield hold,self,self.looktime

    def setvar(self):
        self.stime = now()
        
    def flushstat(self):
        curs = self.conn.cursor()
        curs.execute("insert into %s values('', '%d','%7.2f')"%
                     (self.name,
                      self.stime,
                      self.value))
        curs.close()

class Needfresh(StatVar):
    def __init__(self, conn, mconn, iter):
        StatVar.__init__(self, conn, 'needfresh', statvarstime['needfresh'], iter)
        self.mconn = mconn

    def setvar(self):
        curs = self.mconn.cursor()
        curs.execute("select count(*) from %s where changed=1 && iter=%d"%
                     (dbcfg.get('dbOptions','PagesTableName'),
                      self.iter))
        k = curs.fetchone()
        curs.close()
        self.value = k[0]
        StatVar.setvar(self)

class Freshness(StatVar):
    def __init__(self, conn, mconn, iter):
        StatVar.__init__(self, conn, 'freshness', statvarstime['freshness'], iter)
        self.mconn = mconn

    def setvar(self):
        curs = self.mconn.cursor()
        curs.execute("select count(*) from %s where changed=1 && iter=%d"%
                     (dbcfg.get('dbOptions','PagesTableName'),
                      self.iter))
        k = curs.fetchone()
        curs.close()
        pcount = testcfg.getint('pageOptions','PageCount')
        self.value = ((pcount - k[0])*100.0)/pcount
        StatVar.setvar(self)

class Sumsize(StatVar):
    def __init__(self, conn, iter):
        StatVar.__init__(self, conn, 'sumsize', statvarstime['sumsize'], iter)

    def setvar(self):
        self.value = simvars['SumSize']
        StatVar.setvar(self)

class NowProcessing(StatVar):
    def __init__(self, conn, iter):
        StatVar.__init__(self, conn, 'nowprocessing', statvarstime['nowprocessing'], iter)

    def setvar(self):
        self.value = simvars['NowProcessing']
        StatVar.setvar(self)
#-----------------------------------------------------------------
  
def main():
    global simvars, dbconn, grv, statdbconn
    putfile("-=[ Simulation Log, started %s ]=-"%strftime("%d.%m.%Y %H:%M:%S", localtime()))
    putfile("---[ SimSensor "+__version__+" ]--")
    putfile("-< Used %s configuration file"%(wdir+taskdir+'\\ini\\'+taskinifile))

    for j in range(testcfg.getint('simOptions','Iterations')):
        jj = j+1
        putfile("----[ Run: "+`jj`+" ]------------------------------------------------")

        grv = Random(RANDOMSEED)
        grv.seed(RANDOMSEED+jj)
        putfile("-< Random seed is %s"%(RANDOMSEED+jj))
        
        initialize()

        simvars = {}
        simvars = simvarsdefault.copy()

        c = Changes(dbconn, jj,0)
        activate(c,c.life(),0)
        
        q = Queries(dbconn, jj,0)
        activate(q,q.life(),0)

        nf = Needfresh(statdbconn, dbconn, jj)
        activate(nf,nf.life(),0)

        fr = Freshness(statdbconn, dbconn, jj)
        activate(fr,fr.life(),0)

        ss = Sumsize(statdbconn, jj)
        activate(ss,ss.life(),0)

        np = NowProcessing(statdbconn, jj)
        activate(np,np.life(),0)

        putfile("-< Run simulation")
        simulate(until=testcfg.getint('simOptions','ModelTime'))
        putfile("-< End simulation")
        putfile("--[ Statistics ]-------------------------------------------")
        for key in simvars:
            try:
                simk = simvars[key]
                simt = simk.mean()
            except (AttributeError, TypeError):
                putfile("-< %s = %d"%(key, simvars[key]))
            else:
                putfile("-< %s = %4.1f"%(simk.name, simk.mean()))
        putfile("--[ Statistics ]-------------------------------------------")
        putfile("----[ Run: "+`jj`+" ]------------------------------------------------")
        print "\n"

    putfile("---[ SimSensor "+__version__+" ]--")
    putfile("-=[ Simulation Log, finished %s ]=-"%strftime("%d.%m.%Y %H:%M:%S", localtime()))    
    putfile("\n")
    flog.close()    
    dbconn.close()
    statdbconn.close()

#--[ Main program :) ]------------------------------------------
if __name__ == '__main__':
    main()
