MikroWizard Initial commit | MikroMan Welcome to the world :)

This commit is contained in:
sepehr 2024-07-20 15:48:46 +03:30
commit 8c49b9a55d
96 changed files with 12274 additions and 0 deletions

99
py/mules/data_grabber.py Normal file
View file

@ -0,0 +1,99 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# syslog.py: independent worker process for grabbing data of devices
# MikroWizard.com , Mikrotik router management solution
# Author: sepehr.ha@gmail.com
import time
from libs import util
from libs.db import db_device,db_sysconfig,db_events
from threading import Thread
from libs.red import RedisDB
import netifaces
import json
import queue
import logging
log = logging.getLogger("Data_grabber")
def grab_device_data(timer=2):
all_devices=list(db_device.get_all_device())
num_threads = len(all_devices)
q = queue.Queue()
threads = []
log.info("Data grabber started")
for dev in all_devices:
time.sleep(0.2)
t = Thread(target=util.grab_device_data, args=(dev, q))
t.start()
threads.append(t)
for t in threads:
t.join()
res=[]
totals={
'rx-total':0,
'tx-total':0
}
data=False
for _ in range(num_threads):
qres=q.get()
if not qres.get("reason",False):
data=qres.get("data", None)
if data:
if data.get("rx-total", False):
totals['rx-total']+=data["rx-total"]
if data.get("tx-total", False):
totals["tx-total"]+=data["tx-total"]
res.append(qres)
else:
db_events.connection_event(qres['id'],'Data Puller',qres.get("detail","connection"),"Critical",0,qres.get("reason","problem in data puller"))
keys=["rx-total","tx-total"]
redopts={
"dev_id":'all',
"keys":keys
}
try:
if data:
reddb=RedisDB(redopts)
reddb.dev_create_keys()
reddb.add_dev_data(data)
except Exception as e:
log.error(e)
def get_all_ipv4_addresses():
ips=db_sysconfig.get_sysconfig('all_ip')
ipv4_addresses = []
# Iterate over all network interfaces
for interface in netifaces.interfaces():
# Get all IPv4 addresses associated with the interface
addresses = netifaces.ifaddresses(interface).get(netifaces.AF_INET, [])
# Append IPv4 addresses to the list
for link in addresses:
if '127.0.0.1' in link['addr']:
continue
ipv4_addresses.append(link['addr'])
ipv4_addresses.sort()
ipv4_addresses=json.dumps(ipv4_addresses)
if ips!=ipv4_addresses:
db_sysconfig.update_sysconfig('all_ip',ipv4_addresses)
def main():
while True:
config=db_sysconfig.get_scan_mode().value
get_all_ipv4_addresses()
grab_device_data()
time.sleep(60)
log.info("data grabbing end")
if __name__ == '__main__':
main()

64
py/mules/firmware.py Normal file
View file

@ -0,0 +1,64 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# syslog.py: independent worker process for updating firmware of incomplate update tasks
# MikroWizard.com , Mikrotik router management solution
# Author: sepehr.ha@gmail.com
import time
from libs import util
from libs.db import db_tasks,db_device
import logging
import queue
from threading import Thread
log = logging.getLogger("Firmware")
try:
from libs import utilpro
ISPRO=True
except ImportError:
ISPRO=False
pass
def updater():
task=db_tasks.firmware_service_status()
if not task.status:
log.info("Firmware updater started")
task.status=1
task.save()
try:
devs = list(db_device.Devices.select().where(db_device.Devices.firmware_to_install.is_null(False) & (db_device.Devices.failed_attempt < 4) & ((db_device.Devices.status=='updated' ) | ( db_device.Devices.status=='failed'))))
num_threads = len(devs)
q = queue.Queue()
threads = []
for dev in devs:
if ISPRO:
t = Thread(target=utilpro.update_device, args=(dev,{"version_to_install":dev.firmware_to_install},False, q))
else:
t = Thread(target=util.update_device, args=(dev, q))
t.start()
threads.append(t)
for t in threads:
t.join()
res=[]
for _ in range(num_threads):
qres=q.get()
except Exception as e:
log.error(e)
task.status=0
task.save()
return False
task.status=0
task.save()
return False
def main():
while True:
try:
updater()
except:
pass
time.sleep(60)
if __name__ == '__main__':
main()

238
py/mules/radius.py Normal file
View file

@ -0,0 +1,238 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# radius.py: independent worker process as a radius server
# MikroWizard.com , Mikrotik router management solution
# Author: sepehr.ha@gmail.com
from libs.db.db_device import Devices,EXCLUDED,database
from libs.db import db_sysconfig
import logging
import time
import asyncio
import logging
import traceback
from pyrad.dictionary import Dictionary
from pyrad.server_async import ServerAsync
from pyrad.packet import AccessAccept,AccessReject
from pyrad.server import RemoteHost
from libs.mschap3 import mschap,mppe
from libs.db import db,db_user_group_perm,db_device,db_groups,db_device,db_AA,db_sysconfig
from libs.util import FourcePermToRouter
try:
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
except:
pass
log = logging.getLogger("Radius")
logging.basicConfig(filename="pyrad.log", level="DEBUG",
format="%(asctime)s [%(levelname)-8s] %(message)s")
class RadServer(ServerAsync):
def __init__(self, loop, dictionary):
ServerAsync.__init__(self, loop=loop, dictionary=dictionary,
debug=True)
def verifyMsChapV2(self,pkt,userpwd,group,nthash):
ms_chap_response = pkt['MS-CHAP2-Response'][0]
authenticator_challenge = pkt['MS-CHAP-Challenge'][0]
if len(ms_chap_response)!=50:
raise Exception("Invalid MSCHAPV2-Response attribute length")
nt_response = ms_chap_response[26:50]
peer_challenge = ms_chap_response[2:18]
_user_name = pkt.get(1)[0]
nt_resp = mschap.generate_nt_response_mschap2(
authenticator_challenge,
peer_challenge,
_user_name,
userpwd,
nthash
)
if nt_resp == nt_response:
auth_resp = mschap.generate_authenticator_response(
userpwd,
nt_response,
peer_challenge,
authenticator_challenge,
_user_name,
nthash
)
mppeSendKey, mppeRecvKey = mppe.mppe_chap2_gen_keys(userpwd, nt_response,nthash)
if group:
reply = self.CreateReplyPacket(pkt, **{
"MS-CHAP2-Success": auth_resp.encode(),
"Mikrotik-Group": group,
})
else:
reply = self.CreateReplyPacket(pkt, **{
"MS-CHAP2-Success": auth_resp.encode(),
})
reply.code = AccessAccept
return reply
else:
return False
def send_auth_reject(self,protocol,pkt,addr):
reply = self.CreateReplyPacket(pkt, **{
})
reply.code = AccessReject
reply.error_msg = "User password wrong"
#log failed attempts
protocol.send_response(reply, addr)
def handle_auth_packet(self, protocol, pkt, addr):
# log.error("Attributes: ")
# for attr in pkt.keys():
# log.error("%s: %s" % (attr, pkt[attr]))
try:
tz=int(time.time())
username = pkt['User-Name'][0]
userip=pkt['Calling-Station-Id'][0]
devip=pkt['NAS-IP-Address'][0]
dev=db_device.query_device_by_ip(devip)
if not dev:
self.send_auth_reject(protocol,pkt,addr)
return
u = db.get_user_by_username(username)
if not u:
self.send_auth_reject(protocol,pkt,addr)
db_AA.Auth.add_log(dev.id, 'failed', username , userip , by=None,sessionid=None,timestamp=tz,message="User Not Exist")
return
else:
#get user permision related to device
if not dev:
self.send_auth_reject(protocol, pkt, addr)
db_AA.Auth.add_log(dev.id, 'failed', username, userip, by=None, sessionid=None, timestamp=tz, message="Device Not Exist")
return
force_perms=True if db_sysconfig.get_sysconfig('force_perms')=="True" else False
if force_perms:
dev_groups=db_groups.devgroups(dev.id)
dev_groups_ids=[group.id for group in dev_groups]
dev_groups_ids.append(1)
res=False
if dev and len(dev_groups_ids)>0:
perm=db_user_group_perm.DevUserGroupPermRel.query_permission_by_user_and_device_group(u.id,dev_groups_ids)
res2=False
if len(list(perm))>0:
res2=FourcePermToRouter(dev,perm)
if not res2:
self.send_auth_reject(protocol,pkt,addr)
db_AA.Auth.add_log(dev.id, 'failed', username , userip , by=None,sessionid=None,timestamp=tz,message="Unable to verify group")
return
nthash=u.hash
if force_perms:
reply=self.verifyMsChapV2(pkt,"password",perm[0].perm_id.name,nthash)
else:
reply=self.verifyMsChapV2(pkt,"password",False,nthash)
if reply:
protocol.send_response(reply, addr)
return
db_AA.Auth.add_log(dev.id, 'failed', username , userip , by=None,sessionid=None,timestamp=tz,message="Wrong Password")
self.send_auth_reject(protocol,pkt,addr)
except Exception as e:
print(e)
self.send_auth_reject(protocol,pkt,addr)
#log failed attempts
def handle_acct_packet(self, protocol, pkt, addr):
try:
ts = int(time.time())
dev_ip=pkt['NAS-IP-Address'][0]
dev=db_device.query_device_by_ip(dev_ip)
type=pkt['Acct-Status-Type'][0]
user=pkt['User-Name'][0]
userip=pkt['Calling-Station-Id'][0]
sessionid=pkt['Acct-Session-Id'][0]
if type == 'Start':
db_AA.Auth.add_log(dev.id, 'loggedin', user , userip , None,timestamp=ts,sessionid=sessionid)
elif type == 'Stop':
db_AA.Auth.add_log(dev.id, 'loggedout', user , userip , None,timestamp=ts,sessionid=sessionid)
except Exception as e:
log.error("Error in accounting: ")
log.error(e)
log.error("Received an accounting request")
log.error("Attributes: ")
log.error(pkt.keys())
# for attr in pkt.keys():
# log.error("%s: %s" % (attr, pkt[attr]))
reply = self.CreateReplyPacket(pkt)
protocol.send_response(reply, addr)
def handle_coa_packet(self, protocol, pkt, addr):
log.error("Received an coa request")
log.error("Attributes: ")
for attr in pkt.keys():
log.error("%s: %s" % (attr, pkt[attr]))
reply = self.CreateReplyPacket(pkt)
protocol.send_response(reply, addr)
def handle_disconnect_packet(self, protocol, pkt, addr):
log.error("Received an disconnect request")
log.error("Attributes: ")
for attr in pkt.keys():
log.error("%s: %s" % (attr, pkt[attr]))
reply = self.CreateReplyPacket(pkt)
# COA NAK
reply.code = 45
protocol.send_response(reply, addr)
def main():
# create server and read dictionary
loop = asyncio.get_event_loop()
server = RadServer(loop=loop, dictionary=Dictionary('py/libs/raddic/dictionary'))
secret = db_sysconfig.get_sysconfig('rad_secret')
server.hosts["0.0.0.0"] = RemoteHost("0.0.0.0",
secret.encode(),
"localhost")
try:
# Initialize transports
loop.run_until_complete(
asyncio.ensure_future(
server.initialize_transports(enable_auth=True,
enable_acct=True,
enable_coa=False,
addresses=['0.0.0.0'])))
try:
# start server
loop.run_forever()
except KeyboardInterrupt as k:
pass
# Close transports
loop.run_until_complete(asyncio.ensure_future(
server.deinitialize_transports()))
except Exception as exc:
log.error('Error: ', exc)
log.error('\n'.join(traceback.format_exc().splitlines()))
# Close transports
loop.run_until_complete(asyncio.ensure_future(
server.deinitialize_transports()))
loop.close()
if __name__ == '__main__':
main()

163
py/mules/syslog.py Normal file
View file

@ -0,0 +1,163 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# syslog.py: independent worker process as a syslog server
# MikroWizard.com , Mikrotik router management solution
# Author: sepehr.ha@gmail.com
from math import e
import socketserver
import re
import time
from libs.db import db_device
import logging
from libs.db import db_AA,db_events
log = logging.getLogger("SYSLOG")
from libs import util
try:
from libs import utilpro
ISPRO=True
except ImportError:
ISPRO=False
pass
import socketserver
class SyslogUDPHandler(socketserver.BaseRequestHandler):
def extract_data_from_regex(self,regex,line):
try:
matches = re.finditer(regex, line, re.MULTILINE)
sgroups=[]
for matchNum, match in enumerate(matches, start=1):
for groupNum in range(0, len(match.groups())):
groupNum = groupNum + 1
sgroups.append(match.group(groupNum))
return sgroups
except:
return None
def handle(self):
data = bytes.decode(self.request[0].strip(), encoding="utf-8")
message = str(data)
#get current timestamp
ts = int(time.time())
socket = self.request[1]
dev=db_device.query_device_by_ip(self.client_address[0])
regex=r'(.*),?(info.*|warning|critical) mikrowizard(\d+):.*'
if dev:
info=self.extract_data_from_regex(regex,message)
opts=util.build_api_options(dev)
try:
int(info[2])
if dev and dev.id != int(info[2]):
log.error("Device id mismatch ignoring syslog for ip : {}".format(self.client_address[0]))
except:
log.error("**device id mismatch")
log.error(message)
log.error(self.client_address[0])
log.error("device id mismatch**")
dev=False
pass
if dev and dev.id == int(info[2]) and 'mikrowizard' in message and 'via api' not in message:
if 'system,info,account' in message:
regex = r"user (.*) logged (in|out) from (..*)via.(.*)"
info=self.extract_data_from_regex(regex,message)
users=util.get_local_users(opts)
try:
if info[0] in users:
msg='local'
else:
msg='radius'
if 'logged in' in message:
if 'via api' not in message:
db_AA.Auth.add_log(dev.id, 'loggedin', info[0] , info[2] , info[3],timestamp=ts,message=msg)
elif 'logged out' in message:
if info[0] in users:
db_AA.Auth.add_log(dev.id, 'loggedout', info[0] , info[2] , info[3],timestamp=ts,message=msg)
except Exception as e:
log.error(e)
log.error(message)
elif 'system,error,critical' in message:
if "login failure" in message:
users=util.get_local_users(opts)
regex = r"login failure for user (.*) from (..*)via.(.*)"
info=self.extract_data_from_regex(regex,message)
ts = int(time.time())
if info[0] in users:
msg='local'
else:
msg='radius'
db_AA.Auth.add_log(dev.id, 'failed', info[0] , info[1] , info[2],timestamp=ts,message=msg)
elif "rebooted" in message:
regex=r'system,error,critical mikrowizard\d+: (.*)'
info=self.extract_data_from_regex(regex,message)
db_events.state_event(dev.id, "syslog", "Unexpected Reboot","Critical",1,info[0])
elif 'system,info mikrowizard' in message:
regex= r"system,info mikrowizard\d+: (.*) (changed|added|removed|unscheduled) by (winbox-\d.{1,3}\d\/.*\(winbox\)|mac-msg\(winbox\)|tcp-msg\(winbox\)|ssh|telnet|api|api-ssl|.*\/web|ftp|www-ssl).*:(.*)@(.*) \((.*)\)"
if re.match(regex, message):
info=self.extract_data_from_regex(regex, message)
address=info[4].split('/')
ctype=''
if 'winbox' in info[2]:
ctype='winbox'
if 'tcp' in info[2]:
ctype='winbox-tcp'
elif 'mac' in info[2]:
ctype='winbox-mac'
if 'terminal' in address:
ctype+='/terminal'
elif 'ssh' in info[2]:
ctype='ssh'
elif 'telnet' in info[2]:
ctype='telnet'
elif '/web' in info[2]:
ctype=info[2].split('/')[1] + " " + "({})".format(info[2].split('/')[0])
elif 'api' in info[2]:
ctype='api'
db_AA.Account.add_log(dev.id, info[0], info[1], info[3],message,ctype, address[0], info[5])
elif "rebooted" in message:
db_events.state_event(dev.id, "syslog", "Router Rebooted","info",1,info[0])
else:
regex = r"system,info mikrowizard\d+: (.*) (changed|added|removed|unscheduled) by (.*)"
info=self.extract_data_from_regex(regex,message)
db_AA.Account.add_log(dev.id, info[0], info[1], info[2],message)
elif 'interface,info mikrowizard' in message:
link_regex = r"interface,info mikrowizard\d+: (.*) link (down|up).*"
events=list(db_events.get_events_by_src_and_status("syslog", 0,dev.id).dicts())
if "link down" in message:
info=self.extract_data_from_regex(link_regex,message)
db_events.state_event(dev.id, "syslog", "Link Down: " + info[0],"Warning",0,"Link is down for {}".format(info[0]))
elif "link up" in message:
info=self.extract_data_from_regex(link_regex,message)
util.check_or_fix_event(events,'state',"Link Down: " + info[0])
elif "dhcp,info mikrowizard" in message:
dhcp_regex=r'dhcp,info mikrowizard\d+: (dhcp-client|.*) (deassigned|assigned|.*) (\d+\.\d+\.\d+\.\d+|on.*address)\s*(from|to|$)\s*(.*)'
info=self.extract_data_from_regex(dhcp_regex,message)
if info and "assigned" in message:
db_events.state_event(dev.id, "syslog", "dhcp assigned","info",1,"server {} assigned {} to {}".format(info[0],info[2],info[4]))
elif info and "deassigned" in message:
db_events.state_event(dev.id, "syslog", "dhcp deassigned","info",1,"server {} deassigned {} from {}".format(info[0],info[2],info[4]))
elif info and "dhcp-client" in message:
db_events.state_event(dev.id, "syslog", "dhcp client","info",1,"{} {}".format(info[1],info[2]))
elif "wireless,info mikrowizard" in message:
if ISPRO:
utilpro.wireless_syslog_event(dev ,message)
else:
regex=r'wireless,info mikrowizard\d+: ([0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2}:[0-9A-Fa-f]{2})@(.*): (connected|disconnected), (signal strength|.*)? (-?\d{2})?.*'
info=self.extract_data_from_regex(regex,message)
if info:
strength=""
if len(info)>4:
strength=info[4]
db_events.state_event(dev.id, "syslog", "wireless client", "info", 1, "{} {} {} {} {}".format(info[0], info[1], info[2], info[3],strength))
log.error(len(info))
log.error(message)
else:
log.error(message)
if __name__ == "__main__":
try:
server = socketserver.UDPServer(("0.0.0.0",5014), SyslogUDPHandler)
server.serve_forever(poll_interval=0.5)
except (IOError, SystemExit):
raise

159
py/mules/updater.py Normal file
View file

@ -0,0 +1,159 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# updater.py: independent worker process for updating MikroWizard to latest version
# MikroWizard.com , Mikrotik router management solution
# Author: sepehr.ha@gmail.com
import time
import datetime
from libs import util
from pathlib import Path
from libs.db import db_sysconfig
import requests
import logging
import os
import hashlib
import zipfile
import subprocess
log = logging.getLogger("Updater_mule")
def set_get_install_date():
install_date=False
try:
install_date=db_sysconfig.get_sysconfig('install_date')
except:
pass
if not install_date:
install_date=datetime.datetime.now()
db_sysconfig.set_sysconfig('install_date',install_date.strftime("%Y-%m-%d %H:%M:%S"))
return install_date
# Example usage
def check_sha256(filename, expect):
"""Check if the file with the name "filename" matches the SHA-256 sum
in "expect"."""
h = hashlib.sha256()
# This will raise an exception if the file doesn't exist. Catching
# and handling it is left as an exercise for the reader.
try:
with open(filename, 'rb') as fh:
# Read and hash the file in 4K chunks. Reading the whole
# file at once might consume a lot of memory if it is
# large.
while True:
data = fh.read(4096)
if len(data) == 0:
break
else:
h.update(data)
return expect == h.hexdigest()
except Exception as e:
return False
def extract_zip_reload(filename,dst):
"""Extract the contents of the zip file "filename" to the directory
"dst". Then reload the updated modules."""
with zipfile.ZipFile(filename, 'r') as zip_ref:
zip_ref.extractall(dst)
# run db migrate
dir ="/app/"
cmd = "cd {}; PYTHONPATH={}py PYSRV_CONFIG_PATH={} python3 scripts/dbmigrate.py".format(dir, dir, "/conf/server-conf.json")
p = subprocess.Popen(cmd, shell=True)
(output, err) = p.communicate()
#This makes the wait possible
p_status = p.wait()
#touch server reload file /app/reload
os.remove(filename)
Path('/app/reload').touch()
def main():
while True:
next_hour = (time.time() // 3600 + 1) * 3600
sleep_time = next_hour - time.time()
# Code to be executed every hour
print("Running hourly Update checker ...")
interfaces = util.get_ethernet_wifi_interfaces()
hwid = util.generate_serial_number(interfaces)
username=False
try:
username = db_sysconfig.get_sysconfig('username')
except:
log.error("No username found")
time.sleep(sleep_time)
continue
# util.send_mikrowizard_request(params)
if not username or username.strip()=="":
log.error("No username found")
time.sleep(sleep_time)
continue
install_date=set_get_install_date()
from _version import __version__
#convert install_date string "%Y-%m-%d %H:%M:%S" to datetime
install_date = datetime.datetime.strptime(install_date, "%Y-%m-%d %H:%M:%S").strftime("%Y%m%d")
# convert install_date from "%Y-%m-%d %H:%M:%S" to ""%Y%m%d"" and append to serial_number
hwid += "-"+install_date
params={
"serial_number": hwid,
"username": username.strip(),
"version": __version__
}
res=False
url="http://mikrowizard.com/wp-json/mikrowizard/v1/get_update"
# send post request to server mikrowizard.com with params in json
try:
response = requests.post(url, json=params)
res = response
except:
time.sleep(sleep_time)
continue
# get response from server
try:
if res and res.status_code == 200:
res=res.json()
if 'token' in res:
params={
"token":res['token'],
"file_name":res['filename'],
"username":username.strip()
}
log.info("Update available/Downloading...")
else:
time.sleep(sleep_time)
continue
except Exception as e:
log.error(e)
# check if filename exist in /app/ and checksum is same then dont continue
if check_sha256("/app/"+res['filename'], res['sha256']):
log.error("Checksum match, File exist")
extract_zip_reload("/app/"+res['filename'],"/app/")
time.sleep(sleep_time)
continue
download_url="http://mikrowizard.com/wp-json/mikrowizard/v1/download_update"
# send post request to server mikrowizard.com with params in json
r = requests.post(download_url,json=params,stream=True)
if "invalid" in r.text or r.text=='false':
log.error("Invalid response")
time.sleep(sleep_time)
continue
with open("/app/"+res['filename'], 'wb') as fd:
for chunk in r.iter_content(chunk_size=128):
fd.write(chunk)
if check_sha256("/app/"+res['filename'], res['sha256']):
log.error("Update downloaded : "+"/app/"+res['filename'])
extract_zip_reload("/app/"+res['filename'],"/app/")
else:
log.error("Checksum not match")
os.remove("/app/"+res['filename'])
time.sleep(sleep_time)
if __name__ == '__main__':
main()