index.py 8.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375
  1. """
  2. This is a RESTful interface implemented using Flask micro framework.
  3. The API is driven by configuration that is organized in terms of the monitoring classes
  4. The API is both restful and websocket/socketio enabled.
  5. We designed the classes to be reusable (and powered by labels):
  6. 'monitoring-type':
  7. 'class':'<class-name>'
  8. 'config':<labeled-class-specific-configuration>'
  9. @TODO:
  10. - In order to make this Saas we need to have the configuration be session driven
  11. - Add socketio, so that each section of the dashboard updates independently
  12. """
  13. from flask import Flask, session, request, redirect, Response
  14. from flask.templating import render_template
  15. from flask_session import Session
  16. import time
  17. import sys
  18. import os
  19. import json
  20. import re
  21. import monitor
  22. import Queue
  23. from utils.transport import *
  24. from utils.workers import ThreadManager, Factory
  25. from utils.ml import ML,AnomalyDetection,AnalyzeAnomaly
  26. import utils.params as SYS_ARGS
  27. import atexit
  28. app = Flask(__name__)
  29. app.config['SECRET_KEY'] = '!h8-[0v8]247-4-360'
  30. #app.secret_key = 'A0Zr98j/3yX R~XHH!jmN]LWX=?RT'
  31. PARAMS = SYS_ARGS.PARAMS
  32. f = open(PARAMS['path'])
  33. CONFIG = json.loads(f.read())
  34. f.close()
  35. #
  36. #
  37. #from threading import Thread, RLock
  38. p = CONFIG['store']['args']
  39. class_read = CONFIG['store']['class']['read']
  40. class_write= CONFIG['store']['class']['write']
  41. factory = DataSourceFactory()
  42. # gReader = factory.instance(type=class_read,args=p)
  43. @app.route('/1/get/apps')
  44. def get_apps():
  45. r = []
  46. try:
  47. gReader = factory.instance(type=class_read,args=p)
  48. r = gReader.view('summary/app_names',key=p['uid'])
  49. except Exception,e:
  50. print (e)
  51. return json.dumps(r)
  52. @app.route('/1/get/summary/<id>')
  53. def get_summary(id):
  54. r = []
  55. try:
  56. gReader = factory.instance(type=class_read,args=p)
  57. #if id == 'apps_resources' :
  58. # r = gReader.view('summary/app_resources',key=p['uid'])
  59. #else:
  60. # r = gReader.view('summary/folder_size',key=p['uid'])
  61. id='summary/'+id.strip()
  62. print p
  63. print id
  64. r = gReader.view(id,key=p['uid'])
  65. except Exception,e:
  66. print (e)
  67. return json.dumps(r)
  68. @app.route("/1/sys/usage/trend")
  69. def get_usage_trend():
  70. """
  71. This function returns trends for the 24 most recent observations in the logs
  72. """
  73. r = {}
  74. try:
  75. gReader = factory.instance(type=class_read,args=p)
  76. r = gReader.view('summary/resource_usage_trend',key=p['uid'])
  77. except Exception,e:
  78. print (e)
  79. return json.dumps(r)
  80. @app.route("/1/app/usage/trend")
  81. def get_usage_detail():
  82. """
  83. This function returns trends for the 24 most recent observations in the logs
  84. """
  85. r = {}
  86. try:
  87. gReader = factory.instance(type=class_read,args=p)
  88. r = gReader.view('summary/app_resource_usage_details',key=p['uid'])
  89. except Exception,e:
  90. print (e)
  91. return json.dumps(r)
  92. @app.route('/get/<id>')
  93. def procs(id):
  94. try:
  95. gReader = factory.instance(type=class_read,args=p)
  96. data = gReader.read()
  97. ahandler = AnalyzeAnomaly()
  98. learn = {}
  99. if 'learn' in data :
  100. for row in data['learn'] :
  101. label = row['label']
  102. learn[label] = row
  103. r = {}
  104. for label in data :
  105. if label not in ['learn','folders'] :
  106. index = len(data[label]) - 1
  107. row = data[label][index]
  108. r[label] = row
  109. #
  110. # Let us determine if this is a normal operation or not
  111. # We will update the status of the information ...
  112. #
  113. for row in r[label] :
  114. index = r[label].index(row)
  115. if row['label'] in learn:
  116. id = row['label']
  117. px = ahandler.predict([row],learn[id])
  118. if px :
  119. # row['anomaly'] = px[1]==1
  120. print ""
  121. print label,' *** ',index
  122. row = dict(row,**px)
  123. r[label][index] =row
  124. #
  125. # @TODO:
  126. # Compile a report here that will be sent to the mailing list
  127. #
  128. except Exception, e:
  129. print e
  130. r = []
  131. return json.dumps(r)
  132. """
  133. This function/endpoint will assess n-virtual environments and return the results
  134. @TODO: Should this be stored for future mining (I don't think so but could be wrong)
  135. """
  136. @app.route('/sandbox')
  137. def sandbox():
  138. global CONFIG
  139. if 'sandbox' in CONFIG: #CONFIG['monitor']:
  140. #handler = HANDLERS['sandbox']['class']
  141. #conf = HANDLERS['sandbox']['config']
  142. r = []
  143. # p = Factory.instance('sandbox',CONFIG)
  144. handler = monitor.Sandbox()
  145. conf = CONFIG['sandbox']
  146. for id in conf:
  147. try:
  148. handler.init(conf[id])
  149. r.append (dict(handler.composite(),**{"label":id}))
  150. except Exception,e:
  151. pass
  152. else:
  153. r = []
  154. return json.dumps(r)
  155. @app.route('/trends')
  156. def trends ():
  157. id = request.args.get('id')
  158. app = request.args.get('app').strip()
  159. p = CONFIG['store']['args']
  160. class_read = CONFIG['store']['class']['read']
  161. gReader = factory.instance(type=class_read,args=p)
  162. r = gReader.read()
  163. if id in r:
  164. r = r[id] #--matrix
  165. series = []
  166. for row in r:
  167. series += [item for item in row if str(item['label'])== app]
  168. if len(series) > 12 :
  169. beg = len(series) - 8
  170. series = series[beg:]
  171. return json.dumps(series)
  172. else:
  173. return "[]"
  174. @app.route('/download',methods=['POST'])
  175. def requirements():
  176. stream = request.form['missing']
  177. print stream
  178. stream = "\n".join(json.loads(stream))
  179. headers = {"content-disposition":"attachment; filename=requirements.txt"}
  180. return Response(stream,mimetype='text/plain',headers=headers)
  181. @app.route('/dashboard')
  182. def dashboard():
  183. context = PARAMS['context']
  184. if 'title' in PARAMS :
  185. title = PARAMS['title']
  186. else:
  187. title = 'Dashboard'
  188. apps = []
  189. try:
  190. gReader = factory.instance(type=class_read,args=p)
  191. apps = gReader.view('summary/app_names',key=p['uid'])
  192. except Exception, e:
  193. print (e)
  194. return render_template('dashboard.html',context=context,title=title,app_names=apps)
  195. @app.route('/upgrade')
  196. def upgrade():
  197. context = PARAMS['context']
  198. if 'title' in PARAMS :
  199. title = PARAMS['title']
  200. else:
  201. title = 'Upgrade'
  202. return render_template('upgrade.html',context=context,title=title)
  203. @app.route('/user')
  204. def user():
  205. context = PARAMS['context']
  206. if 'title' in PARAMS :
  207. title = PARAMS['title']
  208. else:
  209. title = 'Upgrade'
  210. return render_template('user.html',context=context,title=title)
  211. """
  212. This function is designed to trigger learning for anomaly detection
  213. @TODO: forward this to a socket i.e non-blocking socket
  214. """
  215. @app.route('/anomalies/get')
  216. def learn():
  217. global CONFIG
  218. p = CONFIG['store']['args']
  219. class_read = CONFIG['store']['class']['read']
  220. gReader = factory.instance(type=class_read,args=p)
  221. d = gReader.read()
  222. if 'learn' in d :
  223. info = d['learn']
  224. del d['learn']
  225. else :
  226. info = []
  227. r = []
  228. if 'id' in request.args:
  229. id = request.args['id']
  230. d = d[id]
  231. params = {}
  232. for item in info:
  233. label = item['label']
  234. params[label] = item
  235. #apps = list(set(ML.Extract(['label'],d)))
  236. r = []
  237. if params :
  238. #
  239. # If we have parameters available
  240. p = AnomalyDetection()
  241. apps = params.keys()
  242. for name in apps :
  243. if name not in params:
  244. continue
  245. _info = params[name]
  246. try:
  247. xo = ML.Filter('label',name,d)
  248. except Exception,e:
  249. xo = []
  250. #print name,e
  251. if len(xo) == 0:
  252. continue
  253. xo = [xo[ len(xo) -1]]
  254. value = p.predict(xo,_info)[0]
  255. if len(value):
  256. report = dict(_info,**{'predicton':value})
  257. r.append(report)
  258. #print app,value
  259. #if value is not None:
  260. # r.append(value)
  261. return json.dumps(r)
  262. """
  263. This function returns anomalies for a given context or group of processes
  264. The information returned is around precision/recall and f-score and parameters
  265. """
  266. @app.route('/anomalies/status')
  267. def anomalies_status():
  268. global CONFIG
  269. p = CONFIG['store']['args']
  270. class_read = CONFIG['store']['class']['read']
  271. gReader = factory.instance(type=class_read,args=p)
  272. d = gReader.read()
  273. if 'learn' in d :
  274. info = d['learn']
  275. del d['learn']
  276. else :
  277. info = []
  278. print info
  279. r = []
  280. if 'id' in request.args:
  281. id = request.args['id']
  282. r = info
  283. return json.dumps(r)
  284. @app.route('/folders')
  285. def get_folders():
  286. global CONFIG
  287. p = CONFIG['store']['args']
  288. class_read = CONFIG['store']['class']['read']
  289. gReader = factory.instance(type=class_read,args=p)
  290. d = gReader.read()
  291. if 'folders' in d:
  292. d = d['folders']
  293. hosts = set([row[0]['id'] for row in d])
  294. m = {}
  295. for id in hosts:
  296. for row in d:
  297. if id == row[0]['id'] :
  298. m[id] = row
  299. d = m.values()
  300. for row in d:
  301. print row[0]['id']
  302. # index = len(d) - 1
  303. # d = d[index]
  304. # m = {}
  305. # for row in d :
  306. # key = row.keys()[0]
  307. # row = row[key]
  308. # if key not in m:
  309. # r.append(row)
  310. # m[key] = len(r) -1
  311. # else:
  312. # index = m[key]
  313. # r[index] = row
  314. # d = r
  315. else:
  316. d = []
  317. return json.dumps(d)
  318. if __name__== '__main__':
  319. # ThreadManager.start(CONFIG)
  320. if 'port' not in SYS_ARGS.PARAMS :
  321. SYS_ARGS.PARAMS['port'] = 8484
  322. PORT = int(SYS_ARGS.PARAMS['port'])
  323. app.run(host='0.0.0.0' ,port=PORT,debug=True,threaded=True)