MikroWizard Initial commit | MikroMan Welcome to the world :)

py/libs/db/db.py

@@ -0,0 +1,170 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# db_user_tasks.py: Models and functions for accsessing db
+# MikroWizard.com , Mikrotik router management solution
+# Author: Tomi.Mickelsson@iki.fi modified by sepehr.ha@gmail.com
+
+from peewee import *
+from playhouse.shortcuts import model_to_dict
+from psycopg2.extensions import ISOLATION_LEVEL_SERIALIZABLE
+
+from flask import abort
+import config
+
+import logging
+log = logging.getLogger("db")
+
+if config.IS_SQLITE:
+    # config.DATABASE_HOST is full path to sqlite file
+    database = SqliteDatabase(config.DATABASE_HOST, pragmas={})
+else:
+    from playhouse.postgres_ext import PostgresqlExtDatabase, ArrayField, BinaryJSONField, BooleanField, JSONField
+    # support for arrays of uuid
+    import psycopg2.extras
+    psycopg2.extras.register_uuid()
+
+    database = PostgresqlExtDatabase(config.DATABASE_NAME,
+        user=config.DATABASE_USER, password=config.DATABASE_PASSWORD,
+        host=config.DATABASE_HOST, port=config.DATABASE_PORT , isolation_level=ISOLATION_LEVEL_SERIALIZABLE)
+
+
+# --------------------------------------------------------------------------
+# Base model and common methods
+
+class BaseModel(Model):
+    """Base class for all database models."""
+
+    # exclude these fields from the serialized dict
+    EXCLUDE_FIELDS = []
+
+    def serialize(self):
+        """Serialize the model into a dict."""
+        d = model_to_dict(self, recurse=False, exclude=self.EXCLUDE_FIELDS)
+        d["id"] = str(d["id"]) # unification: id is always a string
+        return d
+
+    class Meta:
+        database = database
+
+
+def get_object_or_404(model, **kwargs):
+    """Retrieve a single object or abort with 404."""
+
+    try:
+        return model.get(**kwargs)
+    except model.DoesNotExist:
+        log.warning("NO OBJECT {} {}".format(model, kwargs))
+        abort(200)
+
+def get_object_or_none(model, **kwargs):
+    """Retrieve a single object or return None."""
+
+    try:
+        return model.get(**kwargs)
+    except model.DoesNotExist:
+        return None
+
+
+# --------------------------------------------------------------------------
+# USER
+
+class User(BaseModel):
+
+    # Should user.id be an integer or uuid? Both have pros and cons.
+    # Since user.id is sensitive data, I selected uuid here.
+    if not config.IS_SQLITE:
+        id = UUIDField(primary_key=True)
+        id.auto_increment = True # is auto generated by server
+
+    username = TextField()
+    password = TextField()
+    hash = TextField()
+    first_name = TextField()
+    last_name = TextField()
+    role = TextField()
+    email = TextField()
+    adminperms = TextField()
+    if not config.IS_SQLITE:
+        tags = ArrayField(TextField)
+    else:
+        tags = TextField()
+
+    created = DateTimeField()
+    modified = DateTimeField()
+
+    EXCLUDE_FIELDS = [password,hash] # never expose password
+
+
+    def is_superuser(self):
+        return self.role == "superuser"
+
+    def full_name(self):
+        return "{} {}".format(self.first_name, self.last_name or '')
+
+    def serialize(self):
+        """Serialize this object to dict/json."""
+
+        d = super(User, self).serialize()
+
+        # add extra data
+        d["fullname"] = self.full_name()
+        d["tags"] = self.tags or [] # never None
+        return d
+
+    def __str__(self):
+        return "<User {}, {}, role={}>".format(self.id,
+                self.username, self.role)
+
+    class Meta:
+        db_table = 'users'
+
+
+def get_user(uid):
+    """Return user object or throw."""
+    return get_object_or_404(User, id=uid)
+
+
+def get_user_by_username(username):
+    """Return user object or None"""
+
+    if not username:
+        return None
+
+    try:
+        # return User.select().where(User.username == username).get()
+        # case insensitive query
+        if config.IS_SQLITE:
+            sql = "SELECT * FROM users where username = ? LIMIT 1"
+            args = username.lower()
+        else:
+            sql = "SELECT * FROM users where LOWER(username) = LOWER(%s) LIMIT 1"
+            args = (username,)
+        return list(User.raw(sql, args))[0]
+
+    except IndexError:
+        return None
+
+
+def query_users(page=0, limit=1000, search=None):
+    """Return list of users. Desc order"""
+
+    page = int(page or 0)
+    limit = int(limit or 1000)
+
+    q = User.select()
+    if search:
+        search = "%"+search+"%"
+        q = q.where(User.first_name ** search | User.last_name ** search |
+                User.username ** search)
+    q = q.paginate(page, limit).order_by(User.id.desc())
+    return q
+
+
+# --------------------------------------------------------------------------
+
+if __name__ == '__main__':
+
+    # quick adhoc tests
+    logging.basicConfig(level=logging.DEBUG)
+

py/libs/db/db_AA.py

@@ -0,0 +1,143 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# db_snippet.py: Models and functions for accsessing db related to auth and acc
+# MikroWizard.com , Mikrotik router management solution
+# Author: sepehr.ha@gmail.com
+
+
+from calendar import c
+from itertools import count
+from peewee import *
+
+from libs.db.db_device import Devices
+from libs.db.db import User,BaseModel
+import time
+import logging
+log = logging.getLogger("db_AA")
+
+import random
+import string
+# --------------------------------------------------------------------------
+# this model contains two foreign keys to user -- it essentially allows us to
+# model a "many-to-many" relationship between users.  by querying and joining
+# on different columns we can expose who a user is "related to" and who is
+# "related to" a given user
+class Auth(BaseModel):
+    devid = ForeignKeyField(db_column='devid', null=True, model=Devices, to_field='id')
+    ltype = TextField()
+    username = TextField()
+    ip = TextField()
+    sessionid = TextField()
+    by = TextField()
+    started=BigIntegerField()
+    ended=BigIntegerField()
+    message=TextField()
+    created = DateTimeField()
+
+    class Meta:
+        # `indexes` is a tuple of 2-tuples, where the 2-tuples are
+        # a tuple of column names to index and a boolean indicating
+        # whether the index is unique or not.
+        db_table = 'auth'
+
+    def add_log(devid,type,username,ip,by,sessionid=False,timestamp=False,message=None):
+        if type=='failed':
+            rand=''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(8))
+            auth=Auth.select().where(Auth.ltype==type, Auth.username==username.strip())
+            if message=='radius':
+                count=0
+                while(len(list(auth))<1 and count<33):
+                    auth=auth.where(Auth.started > timestamp-2,Auth.started < timestamp+2)
+                    time.sleep(0.3)
+                    count+=1
+            else:
+                auth=False
+            if auth:
+                count=1
+                for a in auth:
+                    if by:
+                        a.by=by.strip()
+                        a.sessionid=str(timestamp+count)+rand
+                    count+=1
+                    a.save()
+            else:
+                if by:
+                    by=by.strip()
+                event=Auth(devid=int(devid), ltype=type, username=username.strip(), ip=ip.strip(), by=by,started=timestamp, ended=timestamp, message=message)
+                event.save()
+        elif type=='loggedin':
+            auth=Auth.select().where(Auth.devid==devid, Auth.ltype==type, Auth.username==username.strip())
+            if sessionid:
+                auth=auth.where(Auth.sessionid==sessionid)
+            else:
+                if message=='radius':
+                    auth=auth.where(Auth.started > timestamp-2,Auth.started < timestamp+2)
+                    count=0
+                    while(len(list(auth))<1 and count<33):
+                        auth=auth.where(Auth.started > timestamp-2,Auth.started < timestamp+2)
+                        time.sleep(0.3)
+                        count+=1
+                        log.error(count)
+                else:
+                    auth=False
+            log.error(auth)
+            if auth:
+                log.error(list(auth))
+            if auth and len(list(auth))>0:
+                auth=list(auth)
+                for a in auth:
+                    if sessionid and not a.sessionid:
+                        a.sessionid=sessionid
+                    if by:
+                        a.by=by.strip()
+                    if message:
+                        a.message=message
+                    a.save()
+            else:
+                if not sessionid:
+                    sessionid=None
+                if by:
+                    by=by.strip()
+                event=Auth(devid=devid,ltype=type,username=username.strip(),ip=ip.strip(),by=by,started=timestamp,sessionid=sessionid,message=message)
+                event.save()
+        else:
+            if sessionid:
+                Auth.update(ended = timestamp).where(Auth.sessionid==sessionid).execute()
+            else:
+                #check if we have same record with type loggedout and same timestamp and same username and if there is not create one
+                if message=='radius':
+                    pass
+                else:
+                    event=Auth(devid=devid, ltype=type, username=username.strip(), ip=ip.strip(), by=by.strip(), ended=timestamp,message=message)
+                    event.save()
+
+class Account(BaseModel):
+    devid = ForeignKeyField(db_column='devid', null=True, model=Devices, to_field='id')
+    username  = TextField()
+    action = TextField()
+    section = TextField()
+    message = TextField()
+    ctype = TextField()
+    address = TextField()
+    config = TextField()
+    created = DateTimeField()
+    class Meta:
+        # `indexes` is a tuple of 2-tuples, where the 2-tuples are
+        # a tuple of column names to index and a boolean indicating
+        # whether the index is unique or not.
+        db_table = 'account'
+
+    def add_log(devid,section,action,username,message,ctype="unknown",address="unknown",config="unknown"):
+        event=Account(devid=devid,section=section.strip(),action=action.strip(),message=message.strip(),username=username.strip(),ctype=ctype.strip(),address=address.strip(),config=config.strip())
+        # print(event.query())
+        event.save()
+
+# --------------------------------------------------------------------------
+
+if __name__ == '__main__':
+
+    # quick adhoc tests
+    logging.basicConfig(level=logging.DEBUG)
+
+ 

py/libs/db/db_backups.py

@@ -0,0 +1,53 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# db_backups.py: Models and functions for accsessing db related to backups
+# MikroWizard.com , Mikrotik router management solution
+# Author: sepehr.ha@gmail.com
+
+from peewee import *
+
+from libs.db.db_device import Devices
+from libs.db.db import User,BaseModel,get_object_or_404
+import datetime
+import logging
+log = logging.getLogger("db_backup")
+    
+
+class Backups(BaseModel):
+    devid = ForeignKeyField(db_column='devid', null=True, model=Devices, to_field='id')
+    dir = TextField()
+    filesize = IntegerField()
+    created = DateTimeField()
+    class Meta:
+        # `indexes` is a tuple of 2-tuples, where the 2-tuples are
+        # a tuple of column names to index and a boolean indicating
+        # whether the index is unique or not.
+        db_table = 'backups'
+
+def get_backup(id):
+    return get_object_or_404(Backups, id=id)
+
+def query_backup_jobs(page=0, limit=1000, search=None , devid=False):
+    page = int(page or 0)
+    limit = int(limit or 1000)
+    q = Backups.select()
+    if search:
+        search = "%"+search+"%"
+        q = q.where(Backups.dir ** search)
+    if devid:
+        q = q.where(Backups.devid == devid) 
+    start_time=datetime.datetime.now()-datetime.timedelta(days=3)
+    q = q.where(Backups.created >= start_time)
+    q = q.paginate(page, limit).order_by(Backups.id.desc())
+    return q
+
+def create(dev,directory,size):
+    backup=Backups(devid=dev.id,dir=directory,filesize=size)
+    backup.save()
+# --------------------------------------------------------------------------
+
+if __name__ == '__main__':
+
+    # quick adhoc tests
+    logging.basicConfig(level=logging.DEBUG)

py/libs/db/db_device.py

@@ -0,0 +1,153 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# db_device.py: Models and functions for accsessing db related to devices
+# MikroWizard.com , Mikrotik router management solution
+# Author: sepehr.ha@gmail.com
+
+from peewee import *
+from libs.db.db import User,BaseModel,database
+
+import logging
+from playhouse.postgres_ext import  BooleanField
+log = logging.getLogger("db")
+
+
+class Devices(BaseModel):
+
+    #id - automatic
+
+    name = TextField()
+    ip = TextField()
+    mac = TextField()
+    details = TextField()
+    uptime = TextField()
+    license = TextField()
+    interface = TextField()
+    user_name = TextField()
+    password = TextField()
+    port = TextField()
+    update_availble = BooleanField()
+    current_firmware = TextField()
+    arch = TextField()
+    sensors = TextField()
+    router_type = TextField()
+    wifi_config = TextField()
+    upgrade_availble = BooleanField()
+    owner = ForeignKeyField(db_column='owner', null=True, model=User, to_field='id')
+    created = DateTimeField()
+    modified = DateTimeField()
+    peer_ip = TextField()
+    failed_attempt = IntegerField()
+    status = TextField()
+    firmware_to_install = TextField()
+    syslog_configured = BooleanField()
+    
+    class Meta:
+        db_table = 'devices'
+
+def get_device(id):
+    q=Devices.select().where(Devices.id == id).dicts().get()
+    return q
+
+def get_devices(ids):
+    q=list(Devices.select().where(Devices.id << ids))
+    return q
+
+def query_device_by_mac(mac):
+    q=Devices.select()
+    try:
+        q=q.where(Devices.serial == mac).get()
+    except:
+        q=False
+
+    return q
+
+def query_device_by_ip(ip):
+    q=Devices.select()
+    try:
+        q=q.where(Devices.ip == ip).get()
+    except:
+        q=False
+    return q
+
+def get_all_device():
+    q=Devices.select()
+    try:
+        q=q
+    except:
+        q=False
+    return q
+
+def get_devices_by_id(ids):
+    q=Devices.select().where(Devices.id << ids)
+    try:
+        q=list(q)
+    except Exception as e :
+        log.error(e)
+        q=False
+    return q
+
+def get_devices_by_id2(ids):
+    q=Devices.select().where(Devices.id << ids)
+    try:
+        q=q
+    except Exception as e :
+        log.error(e)
+        q=False
+    return q
+
+#same with get all devices but we dont return sensetive data
+def get_all_device_api():
+    q=Devices.select(
+        Devices.id,
+        Devices.name ,
+        Devices.ip ,
+        Devices.mac ,
+        Devices.details ,
+        Devices.uptime ,
+        Devices.license ,
+        Devices.interface ,
+        Devices.user_name ,
+        Devices.port ,
+        Devices.update_availble ,
+        Devices.current_firmware ,
+        Devices.arch ,
+        Devices.sensors ,
+        Devices.upgrade_availble ,
+        Devices.owner ,
+        Devices.created ,
+        Devices.modified
+    ).order_by(Devices.id)
+
+    try:
+        q=list(q.dicts())
+    except:
+        q=False
+
+    return q
+
+def update_devices_firmware_status(data):
+    database.execute_sql("SELECT setval('devices_id_seq', MAX(id), true) FROM devices")
+    query=Devices.insert_many(data).on_conflict(conflict_target=Devices.id,update={Devices.update_availble:EXCLUDED.update_availble,Devices.upgrade_availble:EXCLUDED.upgrade_availble,Devices.current_firmware:EXCLUDED.current_firmware,Devices.arch:EXCLUDED.arch})
+    query.execute() 
+    return True
+
+def update_device(devid, user_name, password, ip, peer_ip, name):
+    device=get_device(devid)
+    if not device:
+        return False
+    try:
+        query=Devices.update(user_name=user_name, password=password, ip=ip, peer_ip=peer_ip, name=name).where(Devices.id == devid)
+        query.execute()
+    except:
+        return False
+    return True
+
+# --------------------------------------------------------------------------
+
+if __name__ == '__main__':
+
+    # quick adhoc tests
+    logging.basicConfig(level=logging.DEBUG)
+

py/libs/db/db_events.py

@@ -0,0 +1,126 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# db_events.py: Models and functions for accsessing db related to Events
+# MikroWizard.com , Mikrotik router management solution
+# Author: sepehr.ha@gmail.com
+
+from peewee import *
+ 
+from libs.db.db_device import Devices
+from libs.db.db import BaseModel
+
+import logging
+log = logging.getLogger("db_events")
+
+from playhouse.postgres_ext import  BooleanField
+
+class Events(BaseModel):
+    devid = ForeignKeyField(db_column='devid', null=True, model=Devices, to_field='id')
+    eventtype = TextField()
+    detail = TextField()
+    level = TextField()
+    src = TextField()
+    eventtime = DateTimeField()
+    status = BooleanField()
+    comment = TextField()
+    fixtime = DateTimeField()
+    class Meta:
+        # `indexes` is a tuple of 2-tuples, where the 2-tuples are
+        # a tuple of column names to index and a boolean indicating
+        # whether the index is unique or not.
+        db_table = 'events'
+
+def get_events_by_src_and_status(src, status,devid):
+    return Events.select().where(Events.src==src, Events.status==status, Events.devid==devid)
+
+def fix_event(id):
+    event=Events.get(Events.id==id)
+    event.update(status=1,fixtime='NOW').where(Events.id==event.id).execute()
+
+def connection_event(devid,src,detail,level,status=0,comment=""):
+    #check if we have same event for device before adding new one
+        event=Events.select().where(
+            Events.devid==devid, 
+            Events.eventtype=="connection", 
+            Events.src==src, 
+            Events.detail==detail, 
+            Events.level==level, 
+            Events.status==False)
+        if not event and not status:
+            event=Events(devid=devid, eventtype="connection", detail=detail, level=level, src=src, status=status ,comment=comment)
+            event.save()
+        elif event and status:
+            list(event)[0].update(status=status).execute()
+
+def config_event(devid,src,detail,level,status=0,comment=""):
+    #check if we have same event for device before adding new one
+    event=Events.select().where(
+        Events.devid==devid, 
+        Events.eventtype=="config", 
+        Events.src==src, 
+        Events.detail==detail, 
+        Events.level==level,
+        Events.status==False)
+    if not event and not status:
+        event=Events(devid=devid, eventtype="config", detail=detail, level=level, src=src, status=status, comment=comment)
+        event.save()
+    elif event and status:
+        list(event)[0].update(status=status).execute()
+
+
+
+def firmware_event(devid,src,detail,level,status=0,comment=""):
+    #check if we have same event for device before adding new one
+    event=Events.select().where(
+        Events.devid==devid, 
+        Events.eventtype=="firmware", 
+        Events.src==src, 
+        Events.detail==detail, 
+        Events.level==level,
+        Events.status==False)
+    if not event and not status:
+        event=Events(devid=devid, eventtype="firmware", detail=detail, level=level, src=src, status=status, comment=comment)
+        event.save()
+    elif event and status:
+        list(event)[0].update(status=status).execute()
+
+def health_event(devid, src, detail, level, status=0, comment=""):
+    #check if we have same event for device before adding new one
+    event=Events.select().where(
+        Events.devid==devid,
+        Events.eventtype=="health",
+        Events.src==src,
+        Events.detail==detail,
+        Events.level==level,
+        Events.status==False)
+    if not event and not status:
+        event=Events(devid=devid, eventtype="health", detail=detail, level=level, src=src, status=status, comment=comment)
+        event.save()
+    elif event and status:
+        list(event)[0].update(status=status).execute()
+
+def state_event(devid, src, detail, level, status=0, comment=""):
+    #check if we have same event for device before adding new one
+    event=Events.select().where(
+        Events.devid==devid,
+        Events.eventtype=="state",
+        Events.src==src,
+        Events.detail==detail,
+        Events.level==level,
+        Events.status==False)
+    if not event and not status:
+        event=Events(devid=devid, eventtype="state", detail=detail, level=level, src=src, status=status, comment=comment)
+        event.save()
+    elif event and status:
+        list(event)[0].update(status=status).execute()
+    elif not event and status:
+        event=Events(devid=devid, eventtype="state", detail=detail, level=level, src=src, status=status, comment=comment)
+        event.save()
+
+# --------------------------------------------------------------------------
+
+if __name__ == '__main__':
+
+    # quick adhoc tests
+    logging.basicConfig(level=logging.DEBUG)

py/libs/db/db_firmware.py

@@ -0,0 +1,59 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# db_firmware.py: Models and functions for accsessing db related to Firmware
+# MikroWizard.com , Mikrotik router management solution
+# Author: sepehr.ha@gmail.com
+
+from peewee import *
+
+from libs.db.db import BaseModel,get_object_or_none
+import logging
+log = logging.getLogger("db_firmware")
+
+
+class Firmware(BaseModel):
+    version = TextField()
+    location = TextField()
+    architecture = TextField()
+    sha256 = TextField()
+    created = DateTimeField()
+    class Meta:
+        # `indexes` is a tuple of 2-tuples, where the 2-tuples are
+        # a tuple of column names to index and a boolean indicating
+        # whether the index is unique or not.
+        db_table = 'firmware'
+
+def get_firm(id):
+    return get_object_or_none(Firmware, id=id)
+
+def get_frim_by_version(version,arch):
+    return get_object_or_none(Firmware, version=version, architecture=arch)
+
+def create_perm(datas):
+    for data in datas:
+        perm=Firmware()
+        perm.version = data["version"]
+        perm.location = data["location"]
+        perm.architecture = data["architecture"]
+        perm.sha256 = data["sha256"]
+        perm.save(force_insert=True)
+
+def query_firms(page=0, limit=1000, search=None):
+    page = int(page or 0)
+    limit = int(limit or 1000)
+    q = Firmware.select()
+    if search:
+        search = "%"+search+"%"
+        q = q.where(Firmware.version ** search)
+    q = q.paginate(page, limit).order_by(Firmware.id.desc())
+    return q
+
+# --------------------------------------------------------------------------
+
+if __name__ == '__main__':
+
+    # quick adhoc tests
+    logging.basicConfig(level=logging.DEBUG)
+
+

py/libs/db/db_groups.py

@@ -0,0 +1,158 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# db_group.py: Models and functions for accsessing db related to Device groups
+# MikroWizard.com , Mikrotik router management solution
+# Author: sepehr.ha@gmail.com
+
+from peewee import *
+
+from libs.db.db import User,BaseModel,get_object_or_none
+import logging
+from libs.db.db_device import Devices
+log = logging.getLogger("db_groups")
+
+
+class DevGroups(BaseModel):
+    name = TextField()
+    owner = ForeignKeyField(db_column='owner', null=True,
+                    model=User, to_field='id')
+    created = DateTimeField()
+    modified = DateTimeField()
+
+    class Meta:
+        db_table = 'device_groups'
+
+def get_group(id):
+    return get_object_or_none(DevGroups, id=id)
+
+class DevGroupRel(BaseModel):
+    group_id = ForeignKeyField(db_column='group_id', null=True,
+                    model=DevGroups, to_field='id')
+    device_id = ForeignKeyField(db_column='device_id', null=True,
+                    model=Devices, to_field='id')
+    class Meta:
+        # `indexes` is a tuple of 2-tuples, where the 2-tuples are
+        # a tuple of column names to index and a boolean indicating
+        # whether the index is unique or not.
+        db_table = 'device_groups_devices_rel'
+        indexes = (
+            # Specify a unique multi-column index on from/to-user.
+            (('group_id', 'device_id'), True),
+        )
+
+def create_group(name):
+    try:
+        #check if we have a group with same name
+        group = get_object_or_none(DevGroups,name=name)
+        #if we do, return id
+        if group:
+            return False
+        group = DevGroups.create(name=name)
+    except IntegrityError:
+        return False
+    return group
+    
+def update_group(id, name):
+    group = get_group(id)
+    group.name = name
+    group.save()
+    return group
+
+def add_devices_to_group(group, devids):
+    data=[]
+    for devid in devids:
+        data.append({'group_id': group, 'device_id': devid})
+    res=DevGroupRel.insert_many(data).on_conflict_ignore().execute()
+    return res
+
+#Get groups of device
+def devgroups(devid):
+    return (DevGroups
+            .select()
+            .join(DevGroupRel, on=DevGroupRel.group_id)
+            .where(DevGroupRel.device_id == devid)
+            .order_by(DevGroups.name))
+
+#Get devices of group
+def devs(groupid):
+    return (Devices
+            .select()
+            .join(DevGroupRel, on=DevGroupRel.device_id)
+            .where(DevGroupRel.group_id == groupid)
+            .order_by(Devices.name))
+
+#Get groups of device
+def devgroups_api(devid):
+    return list(DevGroups
+            .select()
+            .join(DevGroupRel, on=DevGroupRel.group_id)
+            .where(DevGroupRel.device_id == devid)
+            .order_by(DevGroups.name).dicts())
+
+#Get devices of group in dict
+def devs(groupid):
+    return list(Devices
+            .select()
+            .join(DevGroupRel, on=DevGroupRel.device_id)
+            .where(DevGroupRel.group_id == groupid)
+            .order_by(Devices.name).dicts())
+
+#Get devices of group
+def devs2(groupid):
+    return list(Devices
+            .select()
+            .join(DevGroupRel, on=DevGroupRel.device_id)
+            .where(DevGroupRel.group_id == groupid)
+            .order_by(Devices.name))
+def get_devs_of_groups(group_ids):
+    try:
+        return list(Devices
+            .select()
+            .join(DevGroupRel, on=DevGroupRel.device_id)
+            .where(DevGroupRel.group_id << group_ids)
+            .order_by(Devices.name))
+    except Exception as e :
+        log.error(e)
+        return []
+        
+#get all groups including devices in each group
+def query_groups_api():
+    t3=DevGroups.alias()
+    q=DevGroups.select(DevGroups.id,DevGroups.name,DevGroups.created,fn.array_agg(DevGroupRel.device_id)).join(DevGroupRel,JOIN.LEFT_OUTER, on=(DevGroupRel.group_id == DevGroups.id)).order_by(DevGroups.id).group_by(DevGroups.id)
+    return list(q.dicts())
+
+def get_groups_by_id(ids):
+    """Return list of unique directors. An example of a raw SQL query."""
+    q=DevGroups.select().where(DevGroups.id << ids)
+    try:
+        q=list(q)
+    except Exception as e :
+        log.error(e)
+        q=False
+    return q
+
+def delete_from_group(devids):
+    delete=DevGroupRel.delete().where(DevGroupRel.device_id << devids).execute()
+    return delete
+
+def delete_device(devid):
+    try:
+
+        delete_from_group([devid])
+        dev = get_object_or_none(Devices, id=devid)
+        dev.delete_instance(recursive=True)
+        return True
+    except Exception as e:
+        log.error(e)
+        return False
+
+
+# --------------------------------------------------------------------------
+
+if __name__ == '__main__':
+
+    # quick adhoc tests
+    logging.basicConfig(level=logging.DEBUG)
+
+

py/libs/db/db_permissions.py

@@ -0,0 +1,74 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# db_permissions.py: Models and functions for accsessing db related to device permisions
+# MikroWizard.com , Mikrotik router management solution
+# Author: sepehr.ha@gmail.com
+
+from peewee import *
+
+import config
+from libs.db.db import BaseModel,get_object_or_none
+
+import logging
+log = logging.getLogger("db_permisions")
+
+
+class Perms(BaseModel):
+    name = TextField()
+    perms = TextField()
+    created = DateTimeField()
+    modified = DateTimeField()
+    class Meta:
+        # `indexes` is a tuple of 2-tuples, where the 2-tuples are
+        # a tuple of column names to index and a boolean indicating
+        # whether the index is unique or not.
+        db_table = 'permissions'
+
+def get_perm(id):
+    return get_object_or_none(Perms, id=id)
+
+def create_perm(name,perms):
+    perm=Perms()
+    perm.name = name
+    perm.perms = perms
+    perm.save(force_insert=True)
+
+def delete_perm(id):
+    if id in [1,2,3]:
+        return False
+    perm = get_object_or_none(Perms, id=id)
+    perm.delete_instance(recursive=True)
+
+def get_perm_by_name(name):
+    if not name:
+        return None
+
+    try:
+        # case insensitive query
+        if config.IS_SQLITE:
+            sql = "SELECT * FROM permissions where name = ? LIMIT 1"
+            args = name.lower()
+        else:
+            sql = "SELECT * FROM permissions where LOWER(name) = LOWER(%s) LIMIT 1"
+            args = (name,)
+        return list(Perms.raw(sql, args))[0]
+
+    except IndexError:
+        return None
+    
+def query_perms(page=0, limit=1000, search=None):
+    page = int(page or 0)
+    limit = int(limit or 1000)
+
+    q = Perms.select()
+    q = q.paginate(page, limit).order_by(Perms.id.desc())
+    return q
+
+# --------------------------------------------------------------------------
+
+if __name__ == '__main__':
+
+    # quick adhoc tests
+    logging.basicConfig(level=logging.DEBUG)
+

py/libs/db/db_sysconfig.py

@@ -0,0 +1,79 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# db_sysconfig.py: Models and functions for accsessing db related to mikrowizard system configs
+# MikroWizard.com , Mikrotik router management solution
+# Author: sepehr.ha@gmail.com
+
+from peewee import *
+
+from libs.db.db import User,BaseModel,get_object_or_404
+import logging
+log = logging.getLogger("db_sysconfig")
+
+class Sysconfig(BaseModel):
+
+    #id - automatic
+
+    key = TextField()
+    value = TextField()
+
+    created_by = ForeignKeyField(db_column='created_by', null=True,
+                    model=User, to_field='id')
+    created = DateTimeField()
+    modified = DateTimeField()
+
+    class Meta:
+        db_table = 'sysconfig'
+
+def get_default_user():
+    return get_object_or_404(Sysconfig, key="default_user")
+
+def get_all():
+    return Sysconfig.select()
+
+def save_all(data):
+    Sysconfig.insert_many(data).on_conflict(conflict_target=['key'], preserve=(Sysconfig.value,Sysconfig.modified)).execute()
+
+def get_default_password():
+    return get_object_or_404(Sysconfig, key="default_password")
+def update_sysconfig(key,value):
+    return Sysconfig.update(value=value).where(Sysconfig.key == key).execute()
+
+def get_scan_mode():
+    return get_object_or_404(Sysconfig, key="scan_mode")
+
+def get_sysconfig(key):
+    return get_object_or_404(Sysconfig, key=key).value
+
+def get_firmware_latest():
+    return get_object_or_404(Sysconfig, key="latest_version")
+
+def get_firmware_action():
+    return get_object_or_404(Sysconfig, key="old_firmware_action")
+
+def get_firmware_old():
+    return get_object_or_404(Sysconfig, key="old_version")
+
+def get_mac_scan_interval():
+    return get_object_or_404(Sysconfig, key="mac_scan_interval")
+
+def get_ip_scan_interval():
+    """Return Movie or throw."""
+    return get_object_or_404(Sysconfig, key="ip_scan_interval")
+
+def update_sysconfig(key,value):
+    return Sysconfig.insert(value=value,key=key).on_conflict(conflict_target=['key'], preserve=['key'], update={'value':value}).execute()                        # firm.version = version
+
+def set_sysconfig(key,value):
+    return Sysconfig.insert(value=value, key=key).on_conflict(conflict_target=['key'], preserve=['key'], update={'value':value}).execute()                        # firm.version = version
+
+
+# --------------------------------------------------------------------------
+
+if __name__ == '__main__':
+
+    # quick adhoc tests
+    logging.basicConfig(level=logging.DEBUG)
+
+

py/libs/db/db_syslog.py

@@ -0,0 +1,47 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# db_syslog.py: Models and functions for accsessing db related to mikrowizard internal logs
+# MikroWizard.com , Mikrotik router management solution
+# Author: sepehr.ha@gmail.com
+
+from peewee import *
+
+from libs.db.db import User,BaseModel
+
+import logging
+log = logging.getLogger("db_syslog")
+
+
+# --------------------------------------------------------------------------
+# this model contains two foreign keys to user -- it essentially allows us to
+# model a "many-to-many" relationship between users.  by querying and joining
+# on different columns we can expose who a user is "related to" and who is
+# "related to" a given user
+class SysLog(BaseModel):
+    user_id = ForeignKeyField(db_column='user_id', null=True, model=User, to_field='id')
+    action = TextField()
+    section = TextField()
+    ip = TextField()
+    agent = TextField()
+    data = TextField()
+    created = DateTimeField()
+    class Meta:
+        # `indexes` is a tuple of 2-tuples, where the 2-tuples are
+        # a tuple of column names to index and a boolean indicating
+        # whether the index is unique or not.
+        db_table = 'syslogs'
+
+
+def add_syslog_event(user_id,section,action,ip,agent,data):
+    event=SysLog(user_id=user_id, section=section, action=action,ip=ip,agent=agent, data=data)
+    event.save()
+
+# --------------------------------------------------------------------------
+
+if __name__ == '__main__':
+
+    # quick adhoc tests
+    logging.basicConfig(level=logging.DEBUG)
+
+

py/libs/db/db_tasks.py

@@ -0,0 +1,73 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# db_tasks.py: Models and functions for accsessing db related to mikrowizard internal logs
+# MikroWizard.com , Mikrotik router management solution
+# Author: sepehr.ha@gmail.com
+
+from peewee import *
+
+from libs.db.db import User,BaseModel
+
+import logging
+log = logging.getLogger("db_tasks")
+
+class Tasks(BaseModel):
+    signal = TextField()
+    starttime = DateTimeField()
+    endtime = DateTimeField()
+    status = BooleanField()
+
+    class Meta:
+        # `indexes` is a tuple of 2-tuples, where the 2-tuples are
+        # a tuple of column names to index and a boolean indicating
+        # whether the index is unique or not.
+        db_table = 'tasks'
+
+#Get groups of device
+def update_check_status():
+    return (Tasks.select().where(Tasks.signal == 100).get())
+
+#Get groups of device
+def update_job_status():
+    return (Tasks.select().where(Tasks.signal == 110).get())
+
+
+#Get groups of device
+def backup_job_status():
+    return (Tasks.select().where(Tasks.signal == 120).get())
+
+#check status of scanner 
+def scanner_job_status():
+    return (Tasks.select().where(Tasks.signal == 130).get())
+
+#check status of downloader 
+def downloader_job_status():
+    return (Tasks.select().where(Tasks.signal == 140).get())
+
+def firmware_service_status():
+    return (Tasks.select().where(Tasks.signal == 150).get())
+
+
+class TaskResults(BaseModel):
+    task_type = TextField()
+    result = DateTimeField()
+    created = DateTimeField()
+
+    class Meta:
+        # `indexes` is a tuple of 2-tuples, where the 2-tuples are
+        # a tuple of column names to index and a boolean indicating
+        # whether the index is unique or not.
+        db_table = 'task_results'
+
+def add_task_result(task_type,result):
+    tr = TaskResults(task_type=task_type, result=result)
+    tr.save()
+
+# --------------------------------------------------------------------------
+
+if __name__ == '__main__':
+
+    # quick adhoc tests
+    logging.basicConfig(level=logging.DEBUG)
+

py/libs/db/db_user_group_perm.py

@@ -0,0 +1,88 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# db_user_group_perm.py: Models and functions for accsessing db related to user groups relation permision
+# MikroWizard.com , Mikrotik router management solution
+# Author: sepehr.ha@gmail.com
+
+
+from peewee import *
+
+from libs.db.db_device import Devices
+from libs.db.db import User,BaseModel
+from libs.db.db_permissions import Perms
+from libs.db.db_groups import DevGroups,DevGroupRel
+
+import logging
+log = logging.getLogger("db_user_group_perm")
+
+
+class DevUserGroupPermRel(BaseModel):
+    user_id = ForeignKeyField(User, related_name='user_id')
+    group_id = ForeignKeyField(DevGroups, related_name='group_id')
+    perm_id = ForeignKeyField(Perms, related_name='perm_id')
+
+    class Meta:
+        db_table = 'user_group_perm_rel'
+
+    def __str__(self):
+        return "DevUserGroupPermRel: user_id: %s, group_id: %s, perm_id: %s" % (self.user_id, self.group_id, self.perm_id)
+
+    def __repr__(self):
+        return "DevUserGroupPermRel: user_id: %s, group_id: %s, perm_id: %s" % (self.user_id, self.group_id, self.perm_id)
+
+    def get_user_devices(uid,group_id):
+        perms=list(DevUserGroupPermRel.select().where(DevUserGroupPermRel.user_id == uid))
+        for perm in perms:
+            if group_id==1 or (perm.group_id.id == 1 and not group_id):
+                return Devices.select()
+            elif perm.group_id.id == 1 and group_id:
+                return Devices.select().join(DevGroupRel).where(DevGroupRel.group_id == group_id)
+        if group_id:
+            return Devices.select().join(DevGroupRel).join(DevUserGroupPermRel,on=(DevUserGroupPermRel.group_id == DevGroupRel.group_id)).where(DevUserGroupPermRel.user_id == uid, DevGroupRel.group_id == group_id)
+        return Devices.select().join(DevGroupRel).join(DevUserGroupPermRel,on=(DevUserGroupPermRel.group_id == DevGroupRel.group_id)).where(DevUserGroupPermRel.user_id == uid)
+
+    def get_user_devices_by_ids(uid,ids):
+        perms=list(DevUserGroupPermRel.select().where(DevUserGroupPermRel.user_id == uid))
+        for perm in perms:
+            if perm.group_id.id == 1:
+                return Devices.select().where(Devices.id << ids)
+        return Devices.select().join(DevGroupRel).join(DevUserGroupPermRel,on=(DevUserGroupPermRel.group_id == DevGroupRel.group_id)).where(DevUserGroupPermRel.user_id == uid,Devices.id << ids)
+    
+    def delete_group(gid):
+        #check if group exists
+        group = DevGroups.select().where(DevGroups.id == gid)
+        if group:
+            try:
+                #First delete records from DevGroupRel
+                delete=DevGroupRel.delete().where(DevGroupRel.group_id == gid).execute()
+                #delete group records from DevUserGroupPermRel
+                delete=DevUserGroupPermRel.delete().where(DevUserGroupPermRel.group_id == gid).execute()
+                delete=DevGroups.delete().where(DevGroups.id == gid).execute()
+                return True
+            except Exception as e:
+                return False
+        return False
+
+
+    def get_user_group_perms(uid):
+        return DevUserGroupPermRel.select().where(DevUserGroupPermRel.user_id == uid)
+    
+    def create_user_group_perm(user_id, group_id, perm_id):
+        return DevUserGroupPermRel.create(user_id=user_id, group_id=group_id, perm_id=perm_id)
+    
+    def query_permission_by_user_and_device_group(uid , devgrupid):
+        q = DevUserGroupPermRel.select().where(DevUserGroupPermRel.group_id  << devgrupid,DevUserGroupPermRel.user_id == uid)
+        return (q)
+    
+    def get_user_group_perm(id):
+        try:
+            return DevUserGroupPermRel.select().where(DevUserGroupPermRel.id == id).get()
+        except:
+            return False
+
+    def delete_user_group_perm(id):
+        try:
+            return DevUserGroupPermRel.delete().where(DevUserGroupPermRel.id == id).execute()
+        except:
+            return False

py/libs/db/db_user_tasks.py

@@ -0,0 +1,128 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# db_user_tasks.py: Models and functions for accsessing db related to user tasks
+# MikroWizard.com , Mikrotik router management solution
+# Author: sepehr.ha@gmail.com
+
+from peewee import *
+from libs.db.db_device import Devices
+from libs.db.db import User,BaseModel,get_object_or_none
+from libs.db.db_groups import DevGroups,get_devs_of_groups
+import logging
+log = logging.getLogger("db_user_tasks")
+
+
+class Snippets(BaseModel):
+
+    name = TextField()
+    description = TextField()
+    content = TextField()
+    created = DateTimeField()
+
+    class Meta:
+        # `indexes` is a tuple of 2-tuples, where the 2-tuples are
+        # a tuple of column names to index and a boolean indicating
+        # whether the index is unique or not.
+        db_table = 'snippets'
+
+def get_snippet_by_name(name):
+    return get_object_or_none(Snippets, name=name)
+
+def get_snippet(id):
+    return get_object_or_none(Snippets, id=id)
+
+def update_snippet(id,name, description, content):
+    snippet = get_object_or_none(Snippets, id=id)
+    snippet.name = name
+    snippet.description = description
+    snippet.content = content
+    snippet.save()
+
+def create_snippet(name, description, content):
+    snippet = Snippets()
+    snippet.name = name
+    snippet.description = description
+    snippet.content = content
+    snippet.save()
+
+def delete_snippet(id):
+    snippet = get_object_or_none(Snippets, id=id)
+    snippet.delete_instance()
+
+class UserTasks(BaseModel):
+
+    name = TextField()
+    description = TextField()
+    desc_cron = TextField()
+    dev_ids = TextField()
+    snippetid = ForeignKeyField(db_column='snippetid', null=True,
+                    model=Snippets, to_field='id')
+    data = TextField()
+    cron = TextField()
+    action = TextField()
+    task_type = TextField()
+    selection_type = TextField()
+    created = DateTimeField()
+
+    class Meta:
+        # `indexes` is a tuple of 2-tuples, where the 2-tuples are
+        # a tuple of column names to index and a boolean indicating
+        # whether the index is unique or not.
+        db_table = 'user_tasks'
+
+    def get_utask_by_id(tid):
+        return get_object_or_none(UserTasks, id=tid)
+
+class TaskDevRel(BaseModel):
+    utask_id = ForeignKeyField(UserTasks, related_name='utask_id')
+    group_id = ForeignKeyField(DevGroups, related_name='group_id')
+    device_id = ForeignKeyField(Devices, related_name='device_id')
+
+    class Meta:
+        db_table = 'task_group_dev_rel'
+
+def get_task_devices(task,return_devs=True):
+    members=[]
+    members=list(TaskDevRel.select().where(TaskDevRel.utask_id == task.id).execute())
+    devs=[]
+    if task.selection_type=='groups':
+        group_ids=[]
+        for mem in members:
+            try:
+                group_ids.append(mem.group_id)
+            except DoesNotExist as err:
+                pass
+        if return_devs:
+            devs=get_devs_of_groups(group_ids) 
+        else:
+            devs=group_ids
+    else:
+        for mem in members:
+            try:
+                devs.append(mem.device_id)
+            except DoesNotExist as err:
+                pass
+    return devs
+
+def add_member_to_task(task_id,members,type='devices'):
+    data=[]
+    for member in members:
+        if type=='groups':
+            data.append({'utask_id': task_id, 'group_id': member})
+        else:
+            data.append({'utask_id': task_id, 'device_id': member})
+    res=TaskDevRel.insert_many(data).on_conflict_ignore().execute()
+    return res
+
+def delete_members(task_id):
+    res=TaskDevRel.delete().where(TaskDevRel.utask_id == task_id).execute()
+    return res
+# --------------------------------------------------------------------------
+
+if __name__ == '__main__':
+
+    # quick adhoc tests
+    logging.basicConfig(level=logging.DEBUG)
+
+