From dfc45ef04f52159ff1a0b8fa5a577bb99f3d34f3 Mon Sep 17 00:00:00 2001 From: Vegard Korvald Date: Thu, 30 Oct 2014 11:43:50 +0100 Subject: [PATCH] added fetchReplays.py --- fetchReplays.py | 277 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100755 fetchReplays.py diff --git a/fetchReplays.py b/fetchReplays.py new file mode 100755 index 0000000..38f0789 --- /dev/null +++ b/fetchReplays.py @@ -0,0 +1,277 @@ +# -*- coding: utf-8 -*- +from suds.client import Client +from suds.transport.http import HttpAuthenticated +from ConfigParser import SafeConfigParser +from email.mime.text import MIMEText +import hashlib +import zlib +import os +import logging +import shutil +import time +import json +import fcntl +import sys +import smtplib + +LOCK = None +DEBUG = False + + +# Authenticates and returns the connection +def connect(usern, passwd): + trans = HttpAuthenticated(username=usern, password=passwd) + url = "https://vidyo-replay1.uio.no/replay/services/VidyoReplayContentManagementService?wsdl" + loc = "https://vidyo-replay1.uio.no/replay/services/VidyoReplayContentManagementService" + return Client(url, location=loc, transport=trans) + + +### +# Lists the .flv-files at dir +### +def list_files(dir): + flvs = [] + + for filename in os.listdir(dir): + if filename.endswith('.flv'): + flvs.append(filename) + return flvs + + +### +# Copies the files to a new dir with a new name. +### +def copy_files(src, dest, delete, h): + tot_size = 0 + video_count = 0 + files = list_files(src) + + for record in sorted(records.records, key=lambda k: k.guid): + for filename in sorted(files): + if not filename.startswith(record.guid): + continue + + new_fname = rename(record, '.flv') + + if not os.path.isfile(dest+new_fname): + # if record.userName == tsdproject/tsdusername + if DEBUG is False: + shutil.copy2(src+record.guid+'.flv', dest+new_fname) + video_count += 1 + # else: + # Should we do something if the file is already there? + + src_sum = checksum(src+filename, h) + dest_sum = checksum(dest+new_fname, h) + if src_sum == dest_sum: + # Overwrites the json file if it already exists + makeJSON(record, dest, dest_sum) + # if delete == True: + # client.service.DeleteRecord(id) + tot_size += os.path.getsize(dest+new_fname) + \ + os.path.getsize(dest+rename(record, '.json')) + else: + logging.warning("checksum error: %s removed", new_fname) + os.remove(dest+new_fname) + files.remove(filename) + break + return tot_size, video_count + + +### +# Returns the checksum. sha256 is default +### +def checksum(vid, h): + f = open(vid, 'rb') + + if h is 'crc32': + return zlib.crc32(f.read()) & 0xffffffff + else: + return hashlib.sha256(f.read()).hexdigest() + + +### +# Makes a JSON-file out of the metadata. +### +def makeJSON(record, dest, checksum): + d = { + 'id': record.id, + 'guid': record.guid, + 'tenantName': record.tenantName, + 'userName': record.userName, + 'userFullName': record.userFullName, + 'dateCreated': record.dateCreated.strftime("%Y-%m-%d %H:%M:%S"), + 'dateCreatedString': record.dateCreatedString, + # 'endTime': record.endTime, + 'duration': record.duration, + 'resolution': record.resolution, + 'framerate': record.framerate, + 'pin': record.pin, + 'recordScope': record.recordScope, + 'title': record.title, + 'roomName': record.roomName, + 'fileLink': record.fileLink, + 'recorderId': record.recorderId, + 'webcast': record.webcast, + 'tags': record.tags, + 'comments': record.comments, + 'locked': record.locked, + 'externalPlaybackLink': record.externalPlaybackLink, + 'fileSize': record.fileSize, + 'checksum': checksum + } + + if DEBUG is False: + f = open(dest+rename(record, '.json'), 'w') + json.dump(d, f) + f.close() + + +### + +# Returns a string with the new filename - 'username-%Y%m%dT%H%M%S+fileEnd' +### +def rename(record, fileEnd): + dt = record.dateCreated + + new_fname = '{}-{}{}'.format( + record.userName, dt.strftime("%Y%m%dT%H%M%S"), fileEnd + ) + + return new_fname + + +### +# Checks if the script is already running +### +def single_instance(path): + global LOCK + LOCK = open(path, 'w+') + + try: + fcntl.lockf(LOCK, fcntl.LOCK_EX | fcntl.LOCK_NB) + return False + except IOError: + logging.warning("the script is already running") + return True + + +### +# Reads the config +### +def read_config(argv): + ini = argv[0] + + if not os.path.isfile(ini): + logging.error("%s does not exist", ini) + return -1 + + parser = SafeConfigParser() + parser.read(ini) + + try: + config = [ + parser.get('auth', 'username'), parser.get('auth', 'password'), + parser.get('config', 'lock'), parser.get('config', 'videoSrc'), + parser.get('config', 'videoDest'), parser.getboolean('config', 'delete'), + ] + except: + logging.error("something wrong in the config: %s", sys.exc_info()[0]) + return -1 + + try: + config.append(parser.get('config', 'hash')) # not mandatory + except Exception: + pass + + return config + + +def email(email): + # Open a plain text file for reading. For this example, assume that + # the text file contains only ASCII characters. + fp = open('fetchReplays.log', 'rb') + # Create a text/plain message + msg = MIMEText(fp.read()) + fp.close() + + # me == the sender's email address + # you == the recipient's email address + msg['Subject'] = 'fetchReplay.log' + msg['From'] = 'noreply@uio.no' + msg['To'] = email + + # Send the message via our own SMTP server, but don't include the + # envelope header. + s = smtplib.SMTP('localhost') + s.sendmail('noreply@uio.no', [email], msg.as_string()) + s.quit() # Import smtplib for the actual sending function + + +## +# Change_ lock, authFile and dest before running the script +# Make an empty lockfile. +# Make an authfile with username (1. line) and password (2. line) +# Make a destination folder to store the copied files +### +if __name__ == "__main__": + form = "[%(asctime)19s - %(levelname)8s - %(funcName)15s()] %(message)s" + + logging.basicConfig( + level=logging.WARNING, filename='fetchReplays.log', + format=form, datefmt="%d.%m.%Y %H:%M:%S" + ) + + if len(sys.argv) is not 3: + logging.error("two arguments (config, mail) required, found %d", len(sys.argv)-1) + sys.exit() + else: + mail = sys.argv[2] + + if '@' not in mail: + logging.error("%s is not an email adr", mail) + sys.exit() + + config = read_config(sys.argv[1:]) + + if config is -1: + email(mail) + sys.exit() + + usern = config[0] + passwd = config[1] + lock = config[2] + src = config[3] + dest = config[4] + delete = config[5] + + if len(config) is 7: + h = config[6] + else: + h = 'sha256' + + if single_instance(lock): + sys.exit() + + start = time.time() + client = connect(usern, passwd) + + try: + records = client.service.RecordsSearch(sortby='date') + except Exception as detail: + logging.error("Exception: %s", detail) + email(mail) + sys.exit() + + tot_size, video_count = copy_files(src, dest, delete, h) + + if os.path.getsize('fetchReplays.log') > 0: + email(mail) + + print "----------------------------------" + print "------------ FINISHED ------------" + print "Number of videos: {}".format(records.allVideosCount) + print "Videos copied: {}".format(video_count) + print "{:.2f} MB copied in {:.2f} seconds".format(tot_size/1024/1024, time.time()-start) + print "----------------------------------" + print "----------------------------------" -- 2.43.5