Tidied up

diff --git a/fetchReplays.py b/fetchReplays.py
index 722fd173d7be4ba5e6f334d767e4deeb751101be..bcd762b92fec04ccb2822ca5e91969d213c47e6e 100755 (executable)
--- a/fetchReplays.py
+++ b/fetchReplays.py
@@ -19,6 +19,7 @@ import base64
 from datetime import date
 # from os.path import join as pj
 
+
 LOCK = None
 
 
@@ -30,18 +31,6 @@ def connect(usern, passwd):
     return Client(url, location=loc, transport=trans, faults=False)
 
 
-'''
-# Returns a list of all the .flv files at dir
-def list_flvs(dir):
-    flvs = []
-
-    for filename in os.listdir(dir):
-        if filename.endswith('.flv'):
-            flvs.append(filename)
-    return flvs
-'''
-
-
 # Check to see if the recording is owned by the "tsd-import" user.
 def check_username(rec):
     username = 'tsd-import'
@@ -52,33 +41,6 @@ def check_username(rec):
         return False
 
 
-'''
-# Returns a dictionary of all the videos that should be moved
-# return {src+record.guid+'.flv': dest+new_fname}
-def list_vids(src, dest, delete, h):
-    # files = sorted(list_flvs(src))
-    sort_recs = sorted(records.records, key=lambda k: k.guid)
-    copy_rv = {}
-
-    for filename, record in zip(files, sort_recs):
-        print filename
-        print "----------------------"
-        print record
-        print "----------------------"
-        time.sleep(10)
-        copy_v = []
-        new_fname = rename(record, '.flv')
-        if not os.path.isfile(pj(dest, new_fname)) and check_username(record):
-            print record
-            os.system('curl -OJ -u tsd-import:cruX1t47 ' + record.fileLink)
-            time.sleep(60)
-            copy_v.append((pj(src, record.guid+'.flv')).encode('utf-8'))
-            copy_v.append(pj(dest, new_fname+'.inflight'))
-            copy_rv[record] = copy_v
-    return copy_rv
-'''
-
-
 # Copies the files from d['src'] to source d['src']: 'dest'
 def copy_files(copy_v, video_count):
     if not os.path.isfile(copy_v[1]) or \
@@ -88,36 +50,14 @@ def copy_files(copy_v, video_count):
     return count
 
 
-'''
-# Iterates over the dictionary and calls several other methods
-# Both the copy_files() and makeJSON() is called from here
-def run(src, dest, client, delete, hashfunc):
-    d = list_vids(src, dest, delete, hashfunc)
-    tot_size = 0
-    video_count = 0
-    # start = time.time()
-
-    for rec, copy_v in d.iteritems():
-        no_dot = copy_v[1].split('.')[0]
-        video_count = copy_files(copy_v, video_count)
-        checks, boo = checksum(copy_v, hashfunc)
-        if boo:
-            os.rename(copy_v[1], no_dot+'.flv')
-            copy_v[1] = no_dot+'.flv'
-            makeJSON(rec, no_dot+'.json', checks, hashfunc)
-            print "Copied: {}".format(copy_v[1])
-            tot_size += os.path.getsize(copy_v[1])
-            if delete is True:
-                client.service.DeleteRecord(id=rec.id)
-        else:
-            os.remove(copy_v[1])
-'''
-
-
 def getVideo(url, username, passwd, dest):
     req = urllib2.Request(url)
     base64string = base64.encodestring('%s:%s' % (username, passwd))[:-1]
     req.add_header("Authorization", "Basic %s" % base64string)
+    proxy_string = "https://%s:%s" % ('proxy.uio.no', '3128')
+    proxy_handler = urllib2.ProxyHandler({"https": proxy_string})
+    opener = urllib2.build_opener(proxy_handler)
+    urllib2.install_opener(opener)
 
     try:
         f = urllib2.urlopen(req)
@@ -143,7 +83,6 @@ def getVideo(url, username, passwd, dest):
 def run(username, passwd, dest, client, delete, hashfunc, records):
     for record in records[1].records:
         if check_username(record):
-            # os.system('cd downloads && curl -OJ -u tsd-import:cruX1t47 ' + record.fileLink)
             filename = getVideo(record.fileLink, username, passwd, dest)
             checks = checksum(dest+filename, hashfunc)
             result = re.search('(.*)_\d\d\d\d', filename)
@@ -177,46 +116,6 @@ def checksum(path, hashfunc):
         return sha256(path)
 
 
-'''
-# Returns the checksum. sha256 is default
-def checksum(core_v, h):
-    def crc32(path):
-        BLOCKSIZE = 65536
-        csum = 0
-        with open(path, 'rb') as afile:
-            buf = afile.read(BLOCKSIZE)
-            while len(buf) > 0:
-                csum = zlib.crc32(buf, csum)
-                buf = afile.read(BLOCKSIZE)
-        return csum & 0xffffffff
-
-    def sha256(path):
-        BLOCKSIZE = 65536
-        hasher = hashlib.sha1()
-        with open(path, 'rb') as afile:
-            buf = afile.read(BLOCKSIZE)
-            while len(buf) > 0:
-                hasher.update(buf)
-                buf = afile.read(BLOCKSIZE)
-        return hasher.hexdigest()
-
-    f1 = core_v[0]
-    f2 = core_v[1]
-
-    if h == 'crc32':
-        sum1 = crc32(f1)
-        sum2 = crc32(f2)
-    else:
-        sum1 = sha256(f1)
-        sum2 = sha256(f2)
-
-    if sum1 == sum2:
-        return sum2, True
-    else:
-        log.warning("checksum error: ", core_v[1])
-'''
-
-
 # Makes a JSON-file from record metadata
 def makeJSON(record, dest, username, checksum, hashfunc):
 
@@ -387,21 +286,14 @@ def read_config(argv):
 
 # Sends the log created from a single run to email
 def email(email, log):
-    # Open a plain text file for reading.  For this example, assume that
-    # the text file contains only ASCII characters.
     fp = open(log, 'rb')
-    # Create a text/plain message
     msg = MIMEText(fp.read())
     fp.close()
 
-    # me == the sender's email address
-    # you == the recipient's email address
     msg['Subject'] = 'fetchReplay.log'
     msg['From'] = 'noreply@uio.no'
     msg['To'] = email
 
-    # Send the message via our own SMTP server, but don't include the
-    # envelope header.
     s = smtplib.SMTP('localhost')
     s.sendmail('noreply@uio.no', [email], msg.as_string())
     s.quit()
@@ -431,7 +323,6 @@ def init_test(argv):
 # Returns the records from VidyoReplay
 # def get_records(usern, passwd, log_name):
 def get_records(client, log_name):
-    # client = connect(config['usern'], config['passwd'])
 
     try:
         records = client.service.RecordsSearch(sortby='date')
@@ -457,8 +348,6 @@ if __name__ == "__main__":
     while 1:
         client = connect(config['usern'], config['passwd'])
         records = get_records(client, log_name)
-        # records = get_records(config['usern'], config['passwd'], log_name)
-        # run(config['src'], config['dest'], client, config['delete'], config['hash'])
         run(config['usern'], config['passwd'], config['dest'], client, config['delete'], config['hash'], records)
 
         if os.path.getsize(log_name+'.tmp') > 0: