web2py.gluon.newcron

1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 4 """ 5 Created by Attila Csipa <web2py@csipa.in.rs> 6 Modified by Massimo Di Pierro <mdipierro@cs.depaul.edu> 7 """ 8 9 import sys 10 import os 11 import threading 12 import logging 13 import time 14 import sched 15 import re 16 import datetime 17 import platform 18 import portalocker 19 import fileutils 20 import cPickle 21 from settings import global_settings 22 23 logger = logging.getLogger("web2py.cron") 24 _cron_stopping = False 25

26 -def absolute_path_link(path):

27 """ 28 Return an absolute path for the destination of a symlink 29 30 """ 31 if os.path.islink(path): 32 link = os.readlink(path) 33 if not os.path.isabs(link): 34 link = os.path.join(os.path.dirname(path), link) 35 else: 36 link = os.path.abspath(path) 37 return link

38

39 -def stopcron():

40 "graceful shutdown of cron" 41 global _cron_stopping 42 _cron_stopping = True

43

44 -class extcron(threading.Thread):

45

46 - def __init__(self, applications_parent):

47 threading.Thread.__init__(self) 48 self.setDaemon(False) 49 self.path = applications_parent 50 crondance(self.path, 'external', startup=True)

51

52 - def run(self):

53 if not _cron_stopping: 54 logger.debug('external cron invocation') 55 crondance(self.path, 'external', startup=False)

56

57 -class hardcron(threading.Thread):

58

59 - def __init__(self, applications_parent):

60 threading.Thread.__init__(self) 61 self.setDaemon(True) 62 self.path = applications_parent 63 crondance(self.path, 'hard', startup=True)

64

65 - def launch(self):

66 if not _cron_stopping: 67 logger.debug('hard cron invocation') 68 crondance(self.path, 'hard', startup = False)

69

70 - def run(self):

71 s = sched.scheduler(time.time, time.sleep) 72 logger.info('Hard cron daemon started') 73 while not _cron_stopping: 74 now = time.time() 75 s.enter(60 - now % 60, 1, self.launch, ()) 76 s.run()

77

78 -class softcron(threading.Thread):

79

80 - def __init__(self, applications_parent):

81 threading.Thread.__init__(self) 82 self.path = applications_parent 83 crondance(self.path, 'soft', startup=True)

84

85 - def run(self):

86 if not _cron_stopping: 87 logger.debug('soft cron invocation') 88 crondance(self.path, 'soft', startup=False)

89

90 -class Token(object):

91

92 - def __init__(self,path):

93 self.path = os.path.join(path, 'cron.master') 94 if not os.path.exists(self.path): 95 fileutils.write_file(self.path, '', 'wb') 96 self.master = None 97 self.now = time.time()

98

99 - def acquire(self,startup=False):

100 """ 101 returns the time when the lock is acquired or 102 None if cron already running 103 104 lock is implemented by writing a pickle (start, stop) in cron.master 105 start is time when cron job starts and stop is time when cron completed 106 stop == 0 if job started but did not yet complete 107 if a cron job started within less than 60 seconds, acquire returns None 108 if a cron job started before 60 seconds and did not stop, 109 a warning is issue "Stale cron.master detected" 110 """ 111 if portalocker.LOCK_EX is None: 112 logger.warning('WEB2PY CRON: Disabled because no file locking') 113 return None 114 self.master = open(self.path,'rb+') 115 try: 116 ret = None 117 portalocker.lock(self.master,portalocker.LOCK_EX) 118 try: 119 (start, stop) = cPickle.load(self.master) 120 except: 121 (start, stop) = (0, 1) 122 if startup or self.now - start > 59.99: 123 ret = self.now 124 if not stop: 125 # this happens if previous cron job longer than 1 minute 126 logger.warning('WEB2PY CRON: Stale cron.master detected') 127 logger.debug('WEB2PY CRON: Acquiring lock') 128 self.master.seek(0) 129 cPickle.dump((self.now,0),self.master) 130 finally: 131 portalocker.unlock(self.master) 132 if not ret: 133 # do this so no need to release 134 self.master.close() 135 return ret

136

137 - def release(self):

138 """ 139 this function writes into cron.master the time when cron job 140 was completed 141 """ 142 if not self.master.closed: 143 portalocker.lock(self.master,portalocker.LOCK_EX) 144 logger.debug('WEB2PY CRON: Releasing cron lock') 145 self.master.seek(0) 146 (start, stop) = cPickle.load(self.master) 147 if start == self.now: # if this is my lock 148 self.master.seek(0) 149 cPickle.dump((self.now,time.time()),self.master) 150 portalocker.unlock(self.master) 151 self.master.close()

152 153

154 -def rangetolist(s, period='min'):

155 retval = [] 156 if s.startswith('*'): 157 if period == 'min': 158 s = s.replace('*', '0-59', 1) 159 elif period == 'hr': 160 s = s.replace('*', '0-23', 1) 161 elif period == 'dom': 162 s = s.replace('*', '1-31', 1) 163 elif period == 'mon': 164 s = s.replace('*', '1-12', 1) 165 elif period == 'dow': 166 s = s.replace('*', '0-6', 1) 167 m = re.compile(r'(\d+)-(\d+)/(\d+)') 168 match = m.match(s) 169 if match: 170 for i in range(int(match.group(1)), int(match.group(2)) + 1): 171 if i % int(match.group(3)) == 0: 172 retval.append(i) 173 return retval

174 175

176 -def parsecronline(line):

177 task = {} 178 if line.startswith('@reboot'): 179 line=line.replace('@reboot', '-1 * * * *') 180 elif line.startswith('@yearly'): 181 line=line.replace('@yearly', '0 0 1 1 *') 182 elif line.startswith('@annually'): 183 line=line.replace('@annually', '0 0 1 1 *') 184 elif line.startswith('@monthly'): 185 line=line.replace('@monthly', '0 0 1 * *') 186 elif line.startswith('@weekly'): 187 line=line.replace('@weekly', '0 0 * * 0') 188 elif line.startswith('@daily'): 189 line=line.replace('@daily', '0 0 * * *') 190 elif line.startswith('@midnight'): 191 line=line.replace('@midnight', '0 0 * * *') 192 elif line.startswith('@hourly'): 193 line=line.replace('@hourly', '0 * * * *') 194 params = line.strip().split(None, 6) 195 if len(params) < 7: 196 return None 197 daysofweek={'sun':0,'mon':1,'tue':2,'wed':3,'thu':4,'fri':5,'sat':6} 198 for (s, id) in zip(params[:5], ['min', 'hr', 'dom', 'mon', 'dow']): 199 if not s in [None, '*']: 200 task[id] = [] 201 vals = s.split(',') 202 for val in vals: 203 if val != '-1' and '-' in val and '/' not in val: 204 val = '%s/1' % val 205 if '/' in val: 206 task[id] += rangetolist(val, id) 207 elif val.isdigit() or val=='-1': 208 task[id].append(int(val)) 209 elif id=='dow' and val[:3].lower() in daysofweek: 210 task[id].append(daysofweek(val[:3].lower())) 211 task['user'] = params[5] 212 task['cmd'] = params[6] 213 return task

214 215

216 -class cronlauncher(threading.Thread):

217

218 - def __init__(self, cmd, shell=True):

219 threading.Thread.__init__(self) 220 if platform.system() == 'Windows': 221 shell = False 222 elif isinstance(cmd,list): 223 cmd = ' '.join(cmd) 224 self.cmd = cmd 225 self.shell = shell

226

227 - def run(self):

228 import subprocess 229 proc = subprocess.Popen(self.cmd, 230 stdin=subprocess.PIPE, 231 stdout=subprocess.PIPE, 232 stderr=subprocess.PIPE, 233 shell=self.shell) 234 (stdoutdata,stderrdata) = proc.communicate() 235 if proc.returncode != 0: 236 logger.warning( 237 'WEB2PY CRON Call returned code %s:\n%s' % \ 238 (proc.returncode, stdoutdata+stderrdata)) 239 else: 240 logger.debug('WEB2PY CRON Call returned success:\n%s' \ 241 % stdoutdata)

242

243 -def crondance(applications_parent, ctype='soft', startup=False):

244 apppath = os.path.join(applications_parent,'applications') 245 cron_path = os.path.join(applications_parent) 246 token = Token(cron_path) 247 cronmaster = token.acquire(startup=startup) 248 if not cronmaster: 249 return 250 now_s = time.localtime() 251 checks=(('min',now_s.tm_min), 252 ('hr',now_s.tm_hour), 253 ('mon',now_s.tm_mon), 254 ('dom',now_s.tm_mday), 255 ('dow',(now_s.tm_wday+1)%7)) 256 257 apps = [x for x in os.listdir(apppath) 258 if os.path.isdir(os.path.join(apppath, x))] 259 260 full_apath_links = set() 261 262 for app in apps: 263 if _cron_stopping: 264 break; 265 apath = os.path.join(apppath,app) 266 267 # if app is a symbolic link to other app, skip it 268 full_apath_link = absolute_path_link(apath) 269 if full_apath_link in full_apath_links: 270 continue 271 else: 272 full_apath_links.add(full_apath_link) 273 274 cronpath = os.path.join(apath, 'cron') 275 crontab = os.path.join(cronpath, 'crontab') 276 if not os.path.exists(crontab): 277 continue 278 try: 279 cronlines = fileutils.readlines_file(crontab, 'rt') 280 lines = [x.strip() for x in cronlines if x.strip() and not x.strip().startswith('#')] 281 tasks = [parsecronline(cline) for cline in lines] 282 except Exception, e: 283 logger.error('WEB2PY CRON: crontab read error %s' % e) 284 continue 285 286 for task in tasks: 287 if _cron_stopping: 288 break; 289 commands = [sys.executable] 290 w2p_path = fileutils.abspath('web2py.py', gluon=True) 291 if os.path.exists(w2p_path): 292 commands.append(w2p_path) 293 if global_settings.applications_parent != global_settings.gluon_parent: 294 commands.extend(('-f', global_settings.applications_parent)) 295 citems = [(k in task and not v in task[k]) for k,v in checks] 296 task_min= task.get('min',[]) 297 if not task: 298 continue 299 elif not startup and task_min == [-1]: 300 continue 301 elif task_min != [-1] and reduce(lambda a,b: a or b, citems): 302 continue 303 logger.info('WEB2PY CRON (%s): %s executing %s in %s at %s' \ 304 % (ctype, app, task.get('cmd'), 305 os.getcwd(), datetime.datetime.now())) 306 action, command, models = False, task['cmd'], '' 307 if command.startswith('**'): 308 (action,models,command) = (True,'',command[2:]) 309 elif command.startswith('*'): 310 (action,models,command) = (True,'-M',command[1:]) 311 else: 312 action=False 313 if action and command.endswith('.py'): 314 commands.extend(('-J', # cron job 315 models, # import models? 316 '-S', app, # app name 317 '-a', '"<recycle>"', # password 318 '-R', command)) # command 319 shell = True 320 elif action: 321 commands.extend(('-J', # cron job 322 models, # import models? 323 '-S', app+'/'+command, # app name 324 '-a', '"<recycle>"')) # password 325 shell = True 326 else: 327 commands = command 328 shell = False 329 try: 330 cronlauncher(commands, shell=shell).start() 331 except Exception, e: 332 logger.warning( 333 'WEB2PY CRON: Execution error for %s: %s' \ 334 % (task.get('cmd'), e)) 335 token.release()

336

Source Code for Module web2py.gluon.newcron