From a87e9b6c5c3662a52140188580d513dfcb4b25d9 Mon Sep 17 00:00:00 2001 From: Enno Rehling Date: Sun, 10 Feb 2019 09:51:06 +0100 Subject: [PATCH 1/5] simplify the orders.queue locking logic, remove it from python --- process/accept-orders.py | 28 ---------------------- process/cron/run-eressea.cron | 21 +++++------------ process/orders-accept | 4 +++- process/orders-process | 4 +++- process/process-orders.py | 44 +++-------------------------------- 5 files changed, 15 insertions(+), 86 deletions(-) diff --git a/process/accept-orders.py b/process/accept-orders.py index 3bf9ea965..69a6ce28f 100755 --- a/process/accept-orders.py +++ b/process/accept-orders.py @@ -60,28 +60,6 @@ writeheaders = True # reject all html email? rejecthtml = True -def unlock_file(filename): - try: - os.unlink(filename+".lock") - except: - print "could not unlock %s.lock, file not found" % filename - -def lock_file(filename): - i = 0 - wait = 1 - if not os.path.exists(filename): - file=open(filename, "w") - file.close() - while True: - try: - os.symlink(filename, filename+".lock") - return - except: - i = i+1 - if i == 5: unlock_file(filename) - sleep(wait) - wait = wait*2 - messages = { "multipart-en" : "ERROR: The orders you sent contain no plaintext. " \ @@ -300,14 +278,12 @@ def accept(game, locale, stream, extend=None): return -1 logger.info("received orders from " + email) # get an available filename - lock_file(gamedir + "/orders.queue") maxdate, filename = available_file(savedir, prefix + email) if filename is None: logger.warning("more than " + str(maxfiles) + " orders from " + email) return -1 # copy the orders to the file text_ok = copy_orders(message, filename, email) - unlock_file(gamedir + "/orders.queue") warning, msg, fail = None, "", False maildate = message.get("Date") @@ -331,10 +307,8 @@ def accept(game, locale, stream, extend=None): os.unlink(filename) savedir = savedir + "/rejected" if not os.path.exists(savedir): os.mkdir(savedir) - lock_file(gamedir + "/orders.queue") maxdate, filename = available_file(savedir, prefix + email) store_message(message, filename) - unlock_file(gamedir + "/orders.queue") fail = True if sendmail and warning is not None: @@ -348,11 +322,9 @@ def accept(game, locale, stream, extend=None): print filename if not fail: - lock_file(gamedir + "/orders.queue") queue = open(gamedir + "/orders.queue", "a") queue.write("email=%s file=%s locale=%s game=%s\n" % (email, filename, locale, game)) queue.close() - unlock_file(gamedir + "/orders.queue") logger.info("done - accepted orders from " + email) diff --git a/process/cron/run-eressea.cron b/process/cron/run-eressea.cron index ecd4c6b11..697cc5e88 100755 --- a/process/cron/run-eressea.cron +++ b/process/cron/run-eressea.cron @@ -19,30 +19,21 @@ mkdir $REPORTS cd $ERESSEA/game-$GAME -# wait for the queue lock to go away -maxt=5 -to=5 -while [ -e orders.queue.lock ] ; do - echo "waiting for orders to finish processing." - sleep $to - let to=$to+$to - let mast=$maxt-1 - [ $maxt -lt 0 ] && break -done - if [ -d test ]; then touch test/execute.lock fi + $BIN/create-orders $GAME $TURN +if [ ! -s $ERESSEA/game-$GAME/orders.$TURN ]; then + echo "server did not create orders for turn $TURN in game $GAME" + exit 2 +fi + $BIN/backup-eressea $GAME $TURN rm -f execute.lock $BIN/run-turn $GAME $TURN touch execute.lock -if [ ! -s $ERESSEA/game-$GAME/orders.$TURN ]; then - echo "server did not create orders for turn $TURN in game $GAME" - exit 2 -fi if [ ! -s $REPORTS/reports.txt ]; then echo "server did not create reports.txt in game $GAME" exit 4 diff --git a/process/orders-accept b/process/orders-accept index 266424a41..1e4dea2aa 100755 --- a/process/orders-accept +++ b/process/orders-accept @@ -1,5 +1,7 @@ #!/bin/sh SCRIPT=$(readlink -f $0) cd $(dirname $SCRIPT) -python accept-orders.py "$@" +lockfile -r3 -l120 orders.queue.lock +python accept-orders.py "$@" +rm -f orders.queue.lock diff --git a/process/orders-process b/process/orders-process index 38b2d1115..4751a94b1 100755 --- a/process/orders-process +++ b/process/orders-process @@ -1,5 +1,7 @@ #!/bin/sh SCRIPT=$(readlink -f $0) cd $(dirname $SCRIPT) -python process-orders.py "$@" +lockfile -r3 -l120 orders.queue.lock +python process-orders.py "$@" +rm -f orders.queue.lock diff --git a/process/process-orders.py b/process/process-orders.py index 30d9ea735..2b3e5c350 100755 --- a/process/process-orders.py +++ b/process/process-orders.py @@ -21,29 +21,6 @@ def pwd_get_email(faction, pwd, pwdfile=None): def split_filename(filename): return os.path.split(filename) -def unlock_file(filename): - try: - unlink(filename+".lock") - except: - print "could not unlock %s.lock, file not found" % filename - raise - -def lock_file(filename): - i = 0 - wait = 1 - if not os.path.exists(filename): - file=open(filename, "w") - file.close() - while True: - try: - symlink(filename, filename+".lock") - return - except: - i = i+1 - if i == 5: - raise - sleep(wait) - wait = wait*2 messages = { "subject-de": "Befehle angekommen", @@ -133,16 +110,12 @@ def echeck(filename, locale, rules): #print "reading password file..." pw_data = EPasswd() try: - pw_data.load_database(os.path.join(game_dir,"eressea.db")) + pw_data.load_database(os.path.join(game_dir, "eressea.db")) except: - pw_data.load_file(os.path.join(game_dir,"passwd")) + pw_data.load_file(os.path.join(game_dir, "passwd")) #print "reading orders.queue..." -# move the queue file to a save space while locking it: -try: - lock_file(queue_file) -except: - exit(0) +# move the queue file to a safe space while locking it: queuefile = open(queue_file, "r") lines = queuefile.readlines() queuefile.close() @@ -150,10 +123,6 @@ queuefile.close() # copy to a temp file tname="/tmp/orders.queue.%s" % str(time()) -try: - lock_file(tname) -except: - exit(0) tmpfile=open(tname, "w") for line in lines: tmpfile.write(line) @@ -161,12 +130,6 @@ tmpfile.close() openlog("orders") -unlink(queue_file) -try: - unlock_file(queue_file) -except: - pass - for line in lines: tokens = split(line[:-1], ' ') dict = {} @@ -221,4 +184,3 @@ for line in lines: closelog() unlink(tname) -unlock_file(tname) From 119e9dbeccbf15e3fc782850ec2a504bb43fc078 Mon Sep 17 00:00:00 2001 From: Enno Rehling Date: Sun, 10 Feb 2019 10:07:08 +0100 Subject: [PATCH 2/5] remove queue after we are done with it --- process/orders-accept | 1 + process/process-orders.py | 1 + 2 files changed, 2 insertions(+) diff --git a/process/orders-accept b/process/orders-accept index 1e4dea2aa..33b3d5ecc 100755 --- a/process/orders-accept +++ b/process/orders-accept @@ -5,3 +5,4 @@ cd $(dirname $SCRIPT) lockfile -r3 -l120 orders.queue.lock python accept-orders.py "$@" rm -f orders.queue.lock + diff --git a/process/process-orders.py b/process/process-orders.py index 2b3e5c350..4d19dde78 100755 --- a/process/process-orders.py +++ b/process/process-orders.py @@ -129,6 +129,7 @@ for line in lines: tmpfile.close() openlog("orders") +unlink(queue_file) for line in lines: tokens = split(line[:-1], ' ') From f052c0542beee37750ccf214e878eac9578f0228 Mon Sep 17 00:00:00 2001 From: Enno Rehling Date: Sun, 17 Feb 2019 11:56:11 +0100 Subject: [PATCH 3/5] fix reports backup, I hope --- process/backup-eressea | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/process/backup-eressea b/process/backup-eressea index 0f81cf53e..12ced938c 100755 --- a/process/backup-eressea +++ b/process/backup-eressea @@ -23,15 +23,16 @@ if [ ! -d backup ] ; then ln -sf ~/backup/eressea/game-$GAME backup fi +if [ -e reports/reports.txt ] ; then + echo "backup reports $TURN, game $GAME" + tar cjf backup/$TURN-reports.tar.bz2 reports + curl -s -n -T backup/$TURN-reports.tar.bz2 https://dav.box.com/dav/Eressea/game-$GAME/$TURN-reports.tar.bz2 +fi files="data/$TURN.dat parteien.full parteien" if [ -e orders.$TURN ]; then files="$files orders.$TURN" fi echo "backup turn $TURN, game $GAME, files: $files" -if [ -d reports ] ; then - tar cjf backup/$TURN-reports.tar.bz2 reports -fi tar cjf backup/$TURN.tar.bz2 $files -echo "uploading game-$GAME/$TURN.tar.bz2" curl -s -n -T backup/$TURN.tar.bz2 https://dav.box.com/dav/Eressea/game-$GAME/$TURN.tar.bz2 #curl -s -n -T eressea.db https://dav.box.com/dav/Eressea/eressea.db From 361a7efe1867fc9d7f5580f4b856128bde5b5a68 Mon Sep 17 00:00:00 2001 From: Enno Rehling Date: Thu, 28 Feb 2019 10:32:08 +0100 Subject: [PATCH 4/5] do not fail on malformed passwords --- process/epasswd.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/process/epasswd.py b/process/epasswd.py index 20109b9ba..496eb8331 100755 --- a/process/epasswd.py +++ b/process/epasswd.py @@ -69,7 +69,12 @@ class EPasswd: def check(self, id, passwd): pw = self.get_passwd(id) if pw[0:4]=='$2a$' or pw[0:4]=='$2y$': - return bcrypt.checkpw(passwd.encode('utf8'), pw.encode('utf8')) + try: + h1 = pw.encode('utf8') + h2 = passwd.encode('utf8') + return bcrypt.checkpw(h1, h2) + except: + return False return pw == passwd def get_passwd(self, id): From d5c1cb67624281ab8176c2166561444da2b7e76e Mon Sep 17 00:00:00 2001 From: Enno Rehling Date: Fri, 1 Mar 2019 11:01:28 +0100 Subject: [PATCH 5/5] stop writing password to logfile --- process/process-orders.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/process/process-orders.py b/process/process-orders.py index 4d19dde78..dade32c02 100755 --- a/process/process-orders.py +++ b/process/process-orders.py @@ -157,7 +157,7 @@ for line in lines: if success: failed = False else: msg = msg + messages["unknown-"+locale] + "\n" msg = msg + "\n" - logfile.write("%s:%s:%s:%s:%s:%s\n" % (ctime(time()), email, game_email, faction, pwd, success)) + logfile.write("%s:%s:%s:%s:%s\n" % (ctime(time()), email, game_email, faction, success)) logfile.close() if failed: