#!/usr/bin/env python3
"DN42 Schema Checker"

from __future__ import print_function

import re
import os
import sys
import time
import argparse
import glob
import urllib.parse
import http.client
import json
import subprocess
import functools

import log

SCHEMA_NAMESPACE = "dn42."
# (dn42)registy url to check local state against
REGISTRY_URL = "git@git.dn42.dev:dn42/registry.git" if not "REG_URL" in os.environ else os.environ["REG_URL"]
REGISTRY_COMMIT = "dn42registry/master"

class SchemaDOM:
    "schema"

    def __init__(self, fn):
        self.name = None
        self.ref = None
        self.primary = None
        self.type = None
        self.src = fn
        f = FileDOM(fn)
        self.schema = self.__parse_schema(f)

    def __parse_schema(self, f):
        schema = {}
        for key, val, _ in f.dom:
            if key == "ref":
                self.ref = val
            elif key == "schema":
                self.name = val

            if key != "key":
                continue

            val = val.split()
            key = val.pop(0)

            schema[key] = set()
            for i in val:
                if i == ">":
                    break

                schema[key].add(i)

            for k, v in schema.items():
                if "schema" in v:
                    self.type = k

                if "primary" in v:
                    self.primary = k
                    schema[k].add("oneline")
                    if "multiline" in v:
                        schema[k].remove("multiline")
                    schema[k].add("single")
                    if "multiple" in v:
                        schema[k].remove("multiple")
                    schema[k].add("required")
                    if "optional" in v:
                        schema[k].remove("optional")
                    if "recommend" in v:
                        schema[k].remove("recommend")
                    if "deprecate" in v:
                        schema[k].remove("deprecate")

                if "oneline" not in v:
                    schema[k].add("multiline")
                if "single" not in v:
                    schema[k].add("multiple")

        return schema

    def check_file(self, f, lookups=None):
        "check file"
        status = "PASS"
        if not f.valid:
            log.error("%s Line 0: File does not parse" % (f.src))
            status = "FAIL"

        for k, v in self.schema.items():
            if "required" in v and k not in f.keys:
                log.error("%s Line 0: Key [%s] not found and is required." % (f.src, k))
                status = "FAIL"
            elif "recommend" in v and k not in f.keys:
                log.notice(
                    "%s Line 0: Key [%s] not found and is recommended." % (f.src, k)
                )
                status = "NOTE"

            if "schema" in v and SCHEMA_NAMESPACE + f.dom[0][0] != self.ref:
                log.error(
                    "%s Line 1: Key [%s] not found and is required as the first line."
                    % (f.src, k)
                )
                status = "FAIL"

            if "single" in v and k in f.keys and len(f.keys[k]) > 1:
                log.warning(
                    "%s Line %d: Key [%s] first defined here and has repeated keys."
                    % (f.src, f.keys[k][0], k)
                )
                for l in f.keys[k][1:]:
                    log.error(
                        "%s Line %d: Key [%s] can only appear once." % (f.src, l, k)
                    )
                    status = "FAIL"

            if "oneline" in v and k in f.multi:
                for l in f.keys[k]:
                    log.error(
                        "%s Line %d: Key [%s] can not have multiple lines."
                        % (f.src, l, k)
                    )
                    status = "FAIL"

        for k, v, l in f.dom:
            if k == self.primary and not f.src.endswith(
                    v.replace("/", "_").replace(" ", "")):
                log.error(
                    "%s Line %d: Primary [%s: %s] does not match filename."
                    % (f.src, l, k, v)
                )
                status = "FAIL"

            if k.startswith("x-"):
                log.info("%s Line %d: Key [%s] is user defined." % (f.src, l, k))

            elif k not in self.schema:
                log.error("%s Line %d: Key [%s] not in schema." % (f.src, l, k))
                status = "FAIL"
                continue
            else:
                if "deprecate" in self.schema[k]:
                    log.info(
                        "%s Line %d: Key [%s] was found and is deprecated."
                        % (f.src, l, k)
                    )
                    status = "INFO"

                if lookups is not None:
                    for o in self.schema[k]:
                        if o.startswith("lookup="):
                            refs = o.split("=", 2)[1].split(",")
                            val = v.split()[0]
                            found = False
                            for ref in refs:
                                if (ref, val) in lookups:
                                    found = True
                            if not found:
                                log.error(
                                    "%s Line %d: Key %s references object %s in %s but does not exist."
                                    % (f.src, l, k, val, refs)
                                )
                                status = "FAIL"
        if status != "FAIL":
            ck = sanity_check(f)
            if ck == "FAIL":
                status = ck

        print("CHECK\t%-54s\t%s\tMNTNERS: %s" % (f.src, status, ",".join(f.mntner)))
        return status


class FileDOM:
    "file"

    def __init__(self, fn):
        self.valid = True
        self.dom = []
        self.keys = {}
        self.multi = {}
        self.mntner = []
        self.schema = None
        self.src = fn

        with open(fn, mode="r", encoding="utf-8") as f:
            dom = []
            keys = {}
            multi = {}
            mntner = []
            last_multi = None

            for lineno, i in enumerate(f.readlines(), 1):
                if re.match(r"[ \t]", i):
                    if len(dom) == 0:
                        log.error("File %s does not parse properly" % (fn))
                        self.valid = False
                        return

                    dom[-1][1] += "\n" + i.strip()

                    if dom[-1][0] not in multi:
                        multi[dom[-1][0]] = []

                    if last_multi is None:
                        multi[dom[-1][0]].append(lineno)
                        last_multi = dom[-1][0]

                else:
                    i = i.split(":")
                    if len(i) < 2:
                        continue

                    dom.append([i[0].strip(), ":".join(i[1:]).strip(), lineno - 1])

                    if i[0].strip() not in keys:
                        keys[i[0].strip()] = []

                    keys[i[0].strip()].append(len(dom) - 1)

                    last_multi = None

                if dom[-1][0] == "mnt-by":
                    mntner.append(dom[-1][1])

        self.dom = dom
        self.keys = keys
        self.multi = multi
        self.mntner = mntner
        self.schema = SCHEMA_NAMESPACE + dom[0][0]

    def __str__(self):
        length = 19
        for i in self.dom:
            if len(i[0]) > length:
                length = len(i[0]) + 2
        s = ""
        for i in self.dom:
            l = i[1].split("\n")

            s += i[0] + ":" + " " * (length - len(i[0])) + l[0] + "\n"
            for m in l[1:]:
                s += " " * (length + 1) + m + "\n"

        return s

    def get(self, key, index=0, default=None):
        "get value"
        if key not in self.keys:
            return default
        if index >= len(self.keys[key]) or index <= -len(self.keys[key]):
            return default

        return self.dom[self.keys[key][index]][1]


def main(infile, schema):
    "main command"
    log.debug("Check File: %s" % (infile))
    f = FileDOM(infile)

    if schema is not None:
        f.schema = schema
    else:
        f.schema = "schema/" + f.schema

    if f.schema is None:
        log.error("Schema is not defined for file")
        return False

    log.debug("Use Schema: %s" % (f.schema))

    s = SchemaDOM(f.schema)
    return s.check_file(f)


def check_schemas(path):
    "check schemas"
    schemas = {}
    for fn in glob.glob(path + "/*"):
        s = SchemaDOM(fn)
        log.info("read schema: %s" % (s.name))
        schemas[s.ref] = s

    ok = True
    c = schemas[SCHEMA_NAMESPACE + "schema"]
    for s in schemas:
        ck = c.check_file(s)
        if not ck:
            ok = False

    return ok


def scan_index(infile, mntner=None):
    "scan index"
    idx = {}
    schemas = {}

    with open(infile, "r") as f:
        for line in f.readlines():
            line = line.split()
            idx[(line[0], line[1])] = line[2:]
            if line[0] == SCHEMA_NAMESPACE + "schema":
                s = SchemaDOM(line[2])
                log.info("read schema: %s" % (s.name))
                schemas[s.ref] = s

    return __scan_index(idx, schemas, mntner)


def scan_files(path, mntner=None, use_file=None):
    "scan files"
    arr = __index_files(path, use_file)

    idx = {}
    schemas = {}

    for dom in arr:
        line = (
            dom.schema,
            dom.src.split("/")[-1].replace("_", "/"),
            dom.src,
            ",".join(dom.mntner),
            dom,
        )

        idx[(line[0], line[1])] = line[2:]
        if line[0] == SCHEMA_NAMESPACE + "schema":
            s = SchemaDOM(line[2])
            schemas[s.ref] = s

    return __scan_index(idx, schemas, mntner, use_file)


def __scan_index(idx, schemas, mntner, use_file=None):
    ok = True
    for k, v in idx.items():
        if use_file is not None and use_file != v[0]:
            continue

        s = schemas.get(k[0], None)
        if s is None:
            log.error("No schema found for %s" % (k[1]))
            print("CHECK\t%-54s\tFAIL\tMNTNERS: UNKNOWN" % (v[2].src))
            ok = "FAIL"

        else:
            mlist = []
            if len(v) > 1:
                mlist = v[1].split(",")

            if mntner is not None and mntner not in mlist:
                continue

            c = v[2]
            ck = s.check_file(c, idx.keys())

            if ck == "INFO" and ok != "FAIL":
                ok = ck
            if ck == "FAIL":
                ok = ck
    return ok


def __index_files(path, use_file=None):
    xlat = {
        "dns/": SCHEMA_NAMESPACE + "domain",
        "inetnum/": SCHEMA_NAMESPACE + "inetnum",
        "inet6num/": SCHEMA_NAMESPACE + "inet6num",
        "route/": SCHEMA_NAMESPACE + "route",
        "route6/": SCHEMA_NAMESPACE + "route6",
        "aut-num/": SCHEMA_NAMESPACE + "aut-num",
        "as-set/": SCHEMA_NAMESPACE + "as-set",
        "as-block/": SCHEMA_NAMESPACE + "as-block",
        "organisation/": SCHEMA_NAMESPACE + "organisation",
        "mntner/": SCHEMA_NAMESPACE + "mntner",
        "person/": SCHEMA_NAMESPACE + "person",
        "role/": SCHEMA_NAMESPACE + "role",
        "tinc-key/": SCHEMA_NAMESPACE + "tinc-key",
        "tinc-keyset/": SCHEMA_NAMESPACE + "tinc-keyset",
        "registry/": SCHEMA_NAMESPACE + "registry",
        "schema/": SCHEMA_NAMESPACE + "schema",
        "key-cert/": SCHEMA_NAMESPACE + "key-cert",
    }

    for root, _, files in os.walk(path):
        ignore = True
        for t in xlat:
            if root + "/" == os.path.join(path, t):
                ignore = False
                break
        if ignore:
            continue

        for f in files:
            if f[0] == ".":
                continue
            dom = FileDOM(os.path.join(root, f))
            yield dom

    if use_file is not None:
        dom = FileDOM(use_file)
        yield dom


def index_files(path):
    "index files"
    idx = __index_files(path)
    for i in idx:
        print("%s\t%s\t%s\t%s" % i)

#                                                       default if found     | not found | on server error: empty dict/str| else
def http_get(server, url, query=None, headers=None) -> list[list[list[str]]] | list[str] | dict[None, None] | str         | bytes:
    "http get"
    if headers is None:
        headers = {}
    if "User-Agent" not in headers:
        headers["User-Agent"] = "curl"
    if "Accept" not in headers:
        headers["Accept"] = "application/json"

    if query is None:
        query = {}

    http_client = http.client.HTTPSConnection(server)

    full_url = url + "?" + urllib.parse.urlencode(query)
    log.debug("GET " + full_url)

    http_client.request("GET", full_url, headers=headers)
    req = http_client.getresponse()
    log.debug("HTTP Response: %d %s" % (req.status, req.reason))

    if "application/json" in req.getheader("Content-Type", "application/json"):
        if req.status > 299:
            return {}
        r = req.read()
        if not isinstance(r, str):
            r = r.decode("utf-8")
        return json.loads(r)

    if req.status > 299:
        return ""

    return req.read()


def find_old(fields=None, filters=None) -> list[list[list[str]]] | list[str]:
    """old find"""
    server = "registry.dn42.us"
    url = "/v1/reg/reg.objects"
    if fields is None:
        fields = []
    if filters is None:
        filters = {}
    query = {
        "fields": ",".join(fields),
        "filter": ",".join([k + "=" + v for k, v in filters.items()]),
    }
    return http_get(server, url, query)

def cache():
    _cache = {}
    def wrapper(f):
        @functools.wraps(f)
        def decorated(*args, **kwargs):
            if str(args)+str(kwargs) in _cache:
                return _cache[str(args)+str(kwargs)]
            ret = f(*args, **kwargs)
            _cache[str(args)+str(kwargs)] = ret
            return ret
        return decorated
    return wrapper


def _run_command(args: list) -> str:
    return subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0].decode()

@cache()
def _get_file_content_upstream(filename: str, commit=REGISTRY_COMMIT) -> [str]:
    log.debug(filename)
    # get the file content at that specific commit
    return _run_command(f"git show {commit}:{filename}".split(" ")).split("\n")

@cache()
def _general_attributes(fields: list, filters: dict) -> list[list]:
    if filters["@type"] == "net":
        obj_type = "inet6num" if ":" in filters["@name"] else "inetnum"
    else:
        obj_type = filters["@type"]
    out = [["@file", f"{obj_type}/{filters['@name']}"], ["@type", filters["@type"]],
           ["@name", filters["@name"]]] if fields == [] else []
    if "@file" in fields:
        out.append(["@file", f"{obj_type}/{filters['@name']}"])
    if "@type" in fields:
        out.append(["@type", filters["@type"]])
    if "@name" in fields:
        out.append(["@name", filters["@name"]])
    return out

@cache()
def _get_values_of_filecontent(filecontent:[str], fields:list=[], filters:dict={}) -> list[list[str]] | list[None]:
    out = _general_attributes(fields, filters)
    _previous_key = ""
    for line in filecontent:
        # empty line -> end of file
        if line == "":
            break
        # empty line but continue with current key
        elif line == "+":
            continue
        if line.startswith(" " * 20):
            if _previous_key == "":
                continue
            if _previous_key in fields:
                out.append([_previous_key, line[20:]])
        elif line.split(":")[0] in fields or fields == []:
            _previous_key = line.split(":")[0]
            out.append([_previous_key, line[20:]])

    return out

def _unexpand_ipv6(addr:str) -> str:
    """unexpand ipv6 with tailing zeros"""
    if addr.endswith("0000"):
        addr = addr[:-4]+":"
    while addr.endswith("0000::"):
        addr = addr.replace("0000::",":")
    
    return "::" if addr == "0000:" else addr 

@cache()
def _get_parent_inetnums(inetnum:str, fields:list=[], family:str=None, commit:str=REGISTRY_COMMIT) -> list[list[str]]:
    subnet_len = int(inetnum.split("/")[1])
    if family == None:
        # contains "." -> is ipv4 presentation
        family = "ipv4" if "." in inetnum else "ipv6"
    if family == "ipv6" and inetnum.startswith("0000"):
        family = "ipv4"
        inetnum = pretty_ip(inetnum.split("/")[0])+ "/" + str(int(inetnum.split("/")[1])-96) 

    out = []
    if family == "ipv4":
        netlevel = 1 
        # cause 0.0.0.0/0 = ::ffff:0:0/96 \subset ::/0
        blk0_6 = find(fields=fields,filters={"@type":"net","cidr":"::/0"}, commit=commit)[0]
        blk0_4 = find(fields=fields,filters={"@type":"net","cidr":"0.0.0.0/0"}, commit=commit)[0]
        if "@netlevel" in fields or fields == []:
            blk0_6 = [["@netlevel",str(1).zfill(3)]] + blk0_6
            blk0_4 = [["@netlevel",str(2).zfill(3)]] + blk0_4
        if "@family" in fields or fields == []:
            blk0_6 = [["@family","ipv6"]] + blk0_6
            blk0_4 = [["@family","ipv4"]] + blk0_4
        # TODO: implement other "@fields"
        
        netlevel += 2
        out=[blk0_6,blk0_4]
        for i in range(1,subnet_len + 1): #ipv4 subnet length
            blk_inet = pretty_ip(inetrange(f'{inetnum.split("/")[0]}/{i}')[0])+f"_{str(i)}"
            blk_content = _get_file_content_upstream(filename=f"data/inetnum/{blk_inet}", commit=commit)
            if blk_content == [""]:
                continue

            blk_filtered = _get_values_of_filecontent(blk_content, fields=fields, filters={"@type":"net","@name": blk_inet})

            if "@netlevel" in fields or fields == []:
                blk_filtered = [["@netlevel",str(netlevel).zfill(3)]] + blk_filtered
            if "@family" in fields or fields == []:
                blk_filtered = [["@family","ipv4"]] + blk_filtered

            netlevel += 1
            out.append(blk_filtered)
    
    elif family == "ipv6":
        netlevel = 1 # cause start counting at 1 ...
        blk0 = find(fields=fields,filters={"@type":"net","cidr":"::/0"}, commit=commit)[0]

        if "@netlevel" in fields or fields == []:
            blk0 = [["@netlevel",str(1).zfill(3)]] + blk0
        if "@family" in fields or fields == []:
            blk0 = [["@family","ipv6"]] + blk0
        # TODO: implement other "@fields"
        
        netlevel += 1
        out=[blk0]
        for i in range(1,subnet_len + 1): #ipv6 subnet length (max=64)
            blk_inet = _unexpand_ipv6(pretty_ip(inet6range(f'{_unexpand_ipv6(inetnum.split("/")[0])}/{i}')[0]))+f"_{str(i)}"
            blk_content = _get_file_content_upstream(filename=f"data/inet6num/{blk_inet}", commit=commit)
            if blk_content == [""]:
                continue

            blk_filtered = _get_values_of_filecontent(blk_content, fields=fields, filters={"@type":"net","@name": blk_inet})

            if "@netlevel" in fields or fields == []:
                blk_filtered = [["@netlevel",str(netlevel).zfill(3)]] + blk_filtered
            if "@family" in fields or fields == []:
                blk_filtered = [["@family","ipv6"]] + blk_filtered

            netlevel += 1
            out.append(blk_filtered)

    return out

@cache()
def _get_parent_as_blocks(as_min:str, as_max:str, fields:list=[], commit:str = REGISTRY_COMMIT) -> list[list[list[str]]]:
    as_min_int, as_max_int = int(as_min[2:]), int(as_max[2:])

    as_blocks = _get_file_content_upstream("data/as-block/", commit = commit)[2:] # returns list of as-block files preceded by "tree $commit:$dir\n" (i.e. 2 "irrelevant" lines)
    out = []
    for block in as_blocks:
        if block =="" or block.startswith("tree"):
            continue
        block_min, block_max = block.split("-")
        block_min_int, block_max_int = int(block_min[2:]), int(block_max[2:])
        if not (block_min_int <= as_min_int and as_max_int <= block_max_int):
            continue
        block_content = _get_file_content_upstream(f"data/as-block/{block}", commit = commit)
        if block_content == [""]: 
            continue #shouldn't happen
        
        block_filtered = _get_values_of_filecontent(block_content, fields=fields, filters={"@type":"as-block","@name":block})
        
        if "@as-min" in fields:
            block_filtered = [["@as-min",block_min]] + block_filtered
        if "@as-max" in fields:
            block_filtered = [["@as-max",block_max]] + block_filtered

        out.append(block_filtered)

    return out



def find_new(fields: list = None, filters: dict = None, commit:str = REGISTRY_COMMIT) -> list[list[list[str]]] | list[None]:
    """find"""
    # filters:
    # @type=...           -> @name:<filename or value of primary key>
    # @type=net           -> @cidr:<filename ~ value of "cidr:" line>        ( =inet{,6}num)
    # @type=net           -> @netmin: "le="+Lnet, @netmax: "ge="+Hnet, @netmask: "lt="+mask ({L,H}net: {lowest,highest} ip in net)
    # @type=route         -> route{,6}:<filename ~ value of "route{,6}:" line>
    # @type=as-block      -> @as-min: "le="+{L,}asn, @as-max: "ge="+{H,}asn   ({L,H}asn: {lowest,highest} asn in as-block)
    #
    # @family=ipv4        -> <>

    # fields:
    # list of keys(and its values) to return of file
    # for inet(6)nums : "@netlevel": "level" of network (::/0=001 -> fd00::/8=002 -> ... )
    # for as-blocks   : "@as-{min,max}" {lowest,highest} allowed asn in block
    # for @family=ipv4: @netlevel, @netmin, @netmax, @uri -> return list of all inet{,6}num+route{,6}
    # general         : @uri: <registry>.<type>.<name> (for type=dns -> <name> = dn42.<domain>,
    #                           type=net: <name> = <ipv6 representation without ":">_<netmask> )
    #                   @name: <filename = value of primary key>
    #                   @file: <@type>/<@name>
    #                   @updated: <date of last serverside update?: like 2024-03-14T16:02:34Z>
    #                   @
    log.debug(f"fields: {fields}, filters: {filters}")
    if fields is None:
        fields = []
    # no/empty filter -> no matches
    if filters is None or filters == {}:
        return []
    if not "dn42registry" in _run_command(["git", "remote"]):
        _run_command(["git", "remote", "add", "dn42registry", REGISTRY_URL])
        ret = subprocess.Popen("git fetch dn42registry master".split(" ")).wait()
        if ret != 0:
            log.error("failed to fetch new remote 'dn42registry' refusing to continue")
            sys.exit(ret)
    if {"@type", "@name"}.issubset(filters.keys()) or {"@type", "cidr"}.issubset(filters.keys()) or {"@type","route"}.issubset(filters.keys()) or {"@type","route6"}.issubset(filters.keys()):
        obj_type = filters["@type"]
        if "@name" in filters.keys():
            obj_name = filters["@name"]
        elif "cidr" in filters.keys():
            obj_name = filters["@name"] = filters["cidr"].replace("/", "_") 
        else: 
            obj_name = filters["route"].replace("/", "_") if "route" in filters.keys() else filters["route6"].replace("/","_")

        if obj_type == "net":
            obj_type = "inet6num" if ":" in obj_name else "inetnum"

        filecontents = _get_file_content_upstream(f"data/{obj_type}/{obj_name}", commit = commit)
        if filecontents == [""]:
            return []

        out = _get_values_of_filecontent(filecontents, fields=fields, filters=filters)
        return [out]
    elif {"@type","route"}.issubset(filters.keys()) or {"@type","route6"}.issubset(filters.keys()):
        obj_type = filters["@type"]

        obj_name = filters["@name"] = filters[filters["@type"]].replace("/", "_") if not "@name" in filters.keys() else filters["@name"]
        if obj_type == "net":
            obj_type = "inet6num" if ":" in obj_name else "inetnum"

        filecontents = _get_file_content_upstream(f"data/{obj_type}/{obj_name}", commit = commit)
        if filecontents == [""]:
            return []

        out = _get_values_of_filecontent(filecontents, fields=fields, filters=filters)
        return [out]

    elif {"@netmin","@netmask"}.issubset(filters.keys()) and filters["@type"]=="net":
        # assumes @netmin="lt=<lowest_ip>", @netmask="<[1..128]: if ipv4: 96+v4_subnetlen>"
        netmin = pretty_ip(filters["@netmin"].split("=")[1])
        inetnum = netmin + "/" + str(int(filters["@netmask"].split("=")[1])-96) if "." in netmin else netmin + "/" + filters["@netmask"].split("=")[1] 

        out = _get_parent_inetnums(inetnum, fields=fields, commit=commit)
        return out
    
    elif {"@as-min","@as-max"}.issubset(filters.keys()) and filters["@type"] == "as-block":
        # assumes @as-min="le=<smallest_asn>", @as-max="ge=<largest_asn>"
        as_min = filters["@as-min"].split("=")[1]
        as_max = filters["@as-max"].split("=")[1]

        out = _get_parent_as_blocks(as_min, as_max, fields, commit = commit)
        return out
    
    elif {"@family"} == filters.keys():
        # works for everything except if "@netlevel" is in fields
        ip_family = filters["@family"]
        obj_type = "inetnum" if ip_family == "ipv4" else "inet6num"

        nets = _get_file_content_upstream(f"data/{obj_type}/", commit = commit)[2:]
        out = []
        for net in nets:
            if net =="" or net.startswith("tree"):
                continue
        
            net_content = _get_file_content_upstream(f"data/{obj_type}/{net}", commit = commit)
            if net_content == [""]: 
                continue #shouldn't happen
            
            net_filtered = _get_values_of_filecontent(net_content, fields=fields, filters={"@type":"net","@name":net})
            cidr = _get_values_of_filecontent(net_content, fields=["cidr"], filters={"@type":"net","@name":net})[0][1]
            net_min, net_max, net_mask = inetrange(cidr) if ip_family == "ipv4" else inet6range(cidr)
            if "@netmin" in fields or fields==[]:
                net_filtered = [["@netmin", net_min]] + net_filtered
            if "@netmin" in fields or fields==[]:
                net_filtered = [["@netmax", net_max]] + net_filtered
            if "@netmask" in fields or fields==[]:
                net_filtered = [["@netmask", net_mask]] + net_filtered
            if "@uri" in fields or fields==[]:
                net_filtered = [["@uri", f"{SCHEMA_NAMESPACE}{obj_type}.{net_min[:-(128-net_mask)//4]}"]] + net_filtered

            out.append(net_filtered)

        obj_type = "route" if ip_family == "ipv4" else "route6"
        routes = _get_file_content_upstream(f"data/{obj_type}/", commit = commit)[2:]

        return out
    else:
        log.warning("not yet implemented")
        raise NotImplementedError(f"find(fields={fields},filters={filters}, commit={commit})")
        # return http_get(server, url, query)

# TODO: rename find_new to find and remove this line ...
find = find_new

def to_num(ip):
    "ipv4 to number"
    ip = [int(i) for i in ip.split(".")]
    return ip[3] + ip[2] * 256 + ip[1] * 256 ** 2 + ip[0] * 256 ** 3


def to_ip(num):
    "number to ipv4"
    return ".".join(
        [str(i) for i in [num >> 24, (num >> 16) & 0xFF, (num >> 8) & 0xFF, num & 0xFF]]
    )


def pretty_ip(addr:str)-> str:
    "pretty ip"
    if addr.startswith("00000000000000000000ffff"):
        addr = addr[-8:]
        addr = int(addr, 16)
        return to_ip(addr)
    return ":".join([addr[i:i + 4] for i in range(0, len(addr), 4)])


def expand_ipv6(addr:str):
    "expand ip6"
    addr = addr.lower()
    if "::" in addr:
        if addr.count("::") > 1:
            return False
        addr = addr.replace("::", ":" * (9 - addr.count(":")))
    if addr.count(":") != 7:
        return False
    return "".join((i.zfill(4) for i in addr.split(":")))


def ip4_to_ip6(ip):
    "ip4 to ip6"
    return "::ffff:%04x:%04x" % (ip >> 16, ip & 0xFFFF)


def inetrange(inet):
    "inet range"
    ip, mask = inet.split("/")
    mask = int(mask)
    ip = to_num(ip) & (0xFFFFFFFF << 32 - mask)
    ip6 = ip4_to_ip6(ip)
    return inet6range("%s/%d" % (ip6, mask + 96))


def inet6range(inet):
    "inet6 range"
    ip, mask = inet.split("/")
    mask = int(mask)

    log.debug(ip)
    ip = expand_ipv6(ip)

    if mask == 128:
        return ip, ip, mask

    offset = int(ip[mask // 4], 16)
    return (
        "%s%x%s"
        % (ip[: mask // 4], offset & (0xF0 >> mask % 4), "0" * (31 - mask // 4)),
        "%s%x%s"
        % (ip[: mask // 4], offset | (0xF >> mask % 4), "f" * (31 - mask // 4)),
        mask,
    )


def test_policy(obj_type, name, mntner, commit:str = REGISTRY_COMMIT):
    "test policy"
    log.debug([obj_type, name, mntner, commit])

    if obj_type in ["organisation",
                    "mntner",
                    "person",
                    "role",
                    "as-set",
                    "schema",
                    "dns",
                    "key-cert",
                    ]:
        if obj_type == "organisation" and not name.startswith("ORG-"):
            log.error("%s does not start with 'ORG-'" % (name))
            return "FAIL"
        elif obj_type == "mntner" and not name.endswith("-MNT"):
            log.error("%s does not end with '-MNT'" % (name))
            return "FAIL"
        elif obj_type == "dns" and not name.endswith(".dn42"):
            log.error("%s does not end with '.dn42'" % (name))
            return "FAIL"
        elif obj_type == "dns" and len(name.strip(".").split(".")) != 2:
            log.error("%s is not a second level domain" % (name))
            return "FAIL"
        elif obj_type in ["person", "role"] and not name.endswith("-DN42"):
            log.error("%s does not end with '-DN42'" % (name))
            return "FAIL"

        lis = find(["mnt-by"], {"@type": obj_type, "@name": name}, commit=commit)
        log.debug(lis)

        if len(lis) == 0:
            log.notice("%s does not currently exist" % (name))
            return "PASS"

        status = "FAIL"
        for o in lis:
            for n in o:
                log.debug(n)
                log.debug(mntner)
                if n[0] == "mnt-by" and n[1] == mntner:
                    status = "PASS"
                    return status

        log.error("%s does not have mnt for object" % (mntner))
        return status

    elif obj_type in ["inetnum", "inet6num"]:
        log.info("Checking inetnum type")
        lis = find(["mnt-by"], {"@type": "net", "cidr": name}, commit=commit)
        log.debug(lis)

        if len(lis) > 0:
            status = "FAIL"
            for o in lis:
                for n in o:
                    if n[0] == "mnt-by" and n[1] == mntner:
                        status = "PASS"
                        log.notice("%s has mnt for current object" % (mntner))
                        return status
            log.error("%s does not have mnt for current object" % (mntner))
            return status

        if obj_type == "inetnum":
            Lnet, Hnet, mask = inetrange(name)
        else:
            Lnet, Hnet, mask = inet6range(name)

        mask = "%03d" % (mask)

        log.info([Lnet, Hnet, mask])
        lis = find(
            ["inetnum", "inet6num", "policy", "@netlevel", "mnt-by", "mnt-lower"],
            {
                "@type": "net",
                "@netmin": "le=" + Lnet,
                "@netmax": "ge=" + Hnet,
                "@netmask": "lt=" + mask,
            },
            commit = commit
        )
        log.debug(lis)

        policy = {}
        select = None
        mntners = []

        for n in lis:
            obj = {}
            for o in n:
                obj[o[0]] = o[1]
                if o[0].startswith("mnt-"):
                    mntners.append(o[1])

            k = obj["@netlevel"]
            policy[k] = obj

            if select is None:
                select = k
            elif select <= k:
                select = k

        if select is None:
            pass

        elif policy.get(select, {}).get("policy", "closed") == "open":
            log.notice("Policy is open for parent object")
            return "PASS"

        # 3. Check if mntner or mnt-lower for any as-block in the tree.
        elif mntner in mntners:
            log.notice("%s has mnt in parent object" % (mntner))
            return "PASS"

    elif obj_type in ["route", "route6"]:
        log.info("Checking route type")
        lis = find(["mnt-by"], {"@type": "route", obj_type: name}, commit = commit)
        log.debug(lis)

        if len(lis) > 0:
            status = "FAIL"
            for o in lis:
                for n in o:
                    if n[0] == "mnt-by" and n[1] == mntner:
                        status = "PASS"
                        log.notice("%s has mnt for current object" % (mntner))
                        return status
            log.error("%s does not have mnt for current object" % (mntner))
            return status

        if obj_type == "route":
            Lnet, Hnet, mask = inetrange(name)
        else:
            Lnet, Hnet, mask = inet6range(name)
        mask = "%03d" % (mask)

        log.info([Lnet, Hnet, mask])
        lis = find(
            ["inetnum", "inet6num", "policy", "@netlevel", "mnt-by", "mnt-lower"],
            {
                "@type": "net",
                "@netmin": "le=" + Lnet,
                "@netmax": "ge=" + Hnet,
                "@netmask": "le=" + mask,
            },
            commit = commit
        )
        log.debug(lis)

        policy = {}
        select = None
        mntners = []

        for n in lis:
            obj = {}
            for o in n:
                obj[o[0]] = o[1]
                if o[0].startswith("mnt-"):
                    mntners.append(o[1])

            k = obj["@netlevel"]
            policy[k] = obj

            if select is None:
                select = k
            elif select <= k:
                select = k

        if select is None:
            pass

        elif policy.get(select, {}).get("policy", "closed") == "open":
            log.notice("Policy is open for parent object")
            return "PASS"

        # 3. Check if mntner or mnt-lower for any as-block in the tree.
        elif mntner in mntners:
            log.notice("%s has mnt in parent object" % (mntner))
            return "PASS"

    elif obj_type == "aut-num":
        if not name.startswith("AS"):
            log.error("%s does not start with AS" % (name))
            return "FAIL"

        # 1. Check if they already have an object
        lis = find(["mnt-by"], {"@type": "aut-num", "@name": name}, commit = commit)
        log.debug(lis)

        if len(lis) > 0:
            status = "FAIL"
            for o in lis:
                for n in o:
                    if n[0] == "mnt-by" and n[1] == mntner:
                        status = "PASS"
                        log.notice("%s has mnt for current object" % (mntner))
                        return status
            log.error("%s does not have mnt for current object" % (mntner))
            return status

        # 2. Check if the as-block has an open policy
        asn = "AS{:0>9}".format(name[2:])
        lis = find(
            ["as-block", "policy", "@as-min", "@as-max", "mnt-by", "mnt-lower"],
            {"@type": "as-block", "@as-min": "le=" + asn, "@as-max": "ge=" + asn},
            commit = commit
        )
        log.info(lis)

        policy = {}
        select = None
        mntners = []

        for n in lis:
            obj = {}
            for o in n:
                obj[o[0]] = o[1]
                if o[0].startswith("mnt-"):
                    mntners.append(o[1])

            k = (obj["@as-min"], obj["@as-max"])
            policy[k] = obj

            if select is None:
                select = k
            elif select[0] <= k[0] or select[1] >= k[1]:
                select = k

        if policy.get(select, {}).get("policy", "closed") == "open":
            log.notice("Policy is open for parent object")
            return "PASS"

        # 3. Check if mntner or mnt-lower for any as-block in the tree.
        elif mntner in mntners:
            log.notice("%s has mnt in parent object" % (mntner))
            return "PASS"

    elif obj_type == "as-block":
        Lname, Hname = name.split("-")
        Lname, Hname = Lname.strip(), Hname.strip()

        if not Lname.startswith("AS") or not Hname.startswith("AS"):
            log.error("%s does not start with AS for min and max" % (name))
            return "FAIL"

        # 1. Check if they already have an object
        lis = find(["mnt-by"], {"@type": "as-block", "@name": name}, commit = commit)
        log.debug(lis)

        if len(lis) > 0:
            status = "FAIL"
            for o in lis:
                for n in o:
                    if n[0] == "mnt-by" and n[1] == mntner:
                        status = "PASS"
                        log.notice("%s has mnt for current object" % (mntner))
                        return status
            log.notice("%s does not have mnt for current object" % (mntner))
            return status

        # 2. Check if the parent as-blocks have an open policy
        Lasn = "AS{:0>9}".format(Lname[2:])
        Hasn = "AS{:0>9}".format(Hname[2:])

        if Lasn > Hasn:
            log.error("%s should come before %s" % (Lname, Hname))

        lis = find(
            ["as-block", "policy", "@as-min", "@as-max", "mnt-by", "mnt-lower"],
            {"@type": "as-block", "@as-min": "le=" + Lasn, "@as-max": "ge=" + Hasn},
            commit = commit
        )
        log.debug(lis)

        policy = {}
        select = None
        mntners = []

        for n in lis:
            obj = {}
            for o in n:
                obj[o[0]] = o[1]
                if o[0].startswith("mnt-"):
                    mntners.append(o[1])

            k = (obj["@as-min"], obj["@as-max"])
            policy[k] = obj

            if select is None:
                select = k
            elif select[0] <= k[0] or select[1] >= k[1]:
                select = k

        # Policy Open only applies to aut-nums. as-blocks must be defined by parent mntners only.
        #
        #   if policy[select]["policy"] == "open":
        #       log.notice("Policy is open for parent object")
        #       return "PASS"

        # 3. Check if mntner or mnt-lower for any as-block in the tree.
        if mntner in mntners:
            log.notice("%s has mnt in parent object" % (mntner))
            return "PASS"

    log.error("%s does not pass checks for %s %s" % (mntner, obj_type, name))
    return "FAIL"


def sanity_check(dom):
    "sanity check"
    ck = "PASS"
    if dom.schema == "dn42.inetnum":
        cidr = dom.get("cidr")
        Lnet, Hnet, _ = inetrange(cidr)
        cidr_range = pretty_ip(Lnet) + "-" + pretty_ip(Hnet)
        file_range = dom.get("inetnum")
        file_range = re.sub(r"\s+", "", file_range, flags=re.UNICODE)

        if cidr_range != file_range:
            log.error(
                "inetnum range [%s] does not match: [%s]" % (file_range, cidr_range)
            )
            ck = "FAIL"

    if dom.schema == "dn42.inet6num":
        cidr = dom.get("cidr")
        log.info(cidr)
        Lnet, Hnet, _ = inet6range(cidr)
        cidr_range = pretty_ip(Lnet) + "-" + pretty_ip(Hnet)
        file_range = dom.get("inet6num")
        file_range = re.sub(r"\s+", "", file_range, flags=re.UNICODE)

        if cidr_range != file_range:
            log.error(
                "inetnum range [%s] does not match: [%s]" % (file_range, cidr_range)
            )
            ck = "FAIL"

    return ck


def get_args():
    """Get and parse command line arguments"""

    parser = argparse.ArgumentParser(
        description="Check Schema. Checks Schema of file for validity"
    )
    parser.add_argument(
        "--merge-output",
        help="Merge stderr into stdout (helps when reading output with pagers) [Default OFF]",
        action="store_true",
    )
    parser.add_argument(
        "-v",
        "--verbose",
        help="Enable verbose output [Default OFF]",
        action="store_true",
    )
    parser.add_argument(
        "-vv",
        "--doubleVerbose",
        help="Enable full verbose output [Default OFF]",
        action="store_true",
    )

    subparsers = parser.add_subparsers(help="sub-command help", dest="command")

    parser_file = subparsers.add_parser("check-file", help="Process a specific file")
    parser_file.add_argument(
        "-s",
        "--use-schema",
        nargs="?",
        help="Override schema to validate [Default None]",
        action="store",
    )
    parser_file.add_argument("infile", nargs="?", help="File to check", type=str)

    parser_schema = subparsers.add_parser("check-schemas", help="Validate all schemas")
    parser_schema.add_argument("path", nargs="?", help="Path for schemas", type=str)

    parser_index = subparsers.add_parser("index", help="Generate index")
    parser_index.add_argument("path", nargs="?", help="Path for dn42 data", type=str)

    parser_scanindex = subparsers.add_parser(
        "scan-index", help="Validate files in index"
    )
    parser_scanindex.add_argument(
        "infile", nargs="?", help="Index file to scan", type=str
    )
    parser_scanindex.add_argument(
        "-m",
        "--use-mntner",
        nargs="?",
        help="Only scan files that has MNT [Default None]",
        action="store",
    )

    parser_scan = subparsers.add_parser("scan", help="Validate files in index")
    parser_scan.add_argument("path", nargs="?", help="Path for dn42 data", type=str)
    parser_scan.add_argument(
        "-m",
        "--use-mntner",
        nargs="?",
        help="Only scan files that has a matching MNT [Default None]",
        action="store",
    )
    parser_scan.add_argument(
        "-f",
        "--use-file",
        nargs="?",
        help="Only scan file given [Default None]",
        action="store",
    )

    parser_fmt = subparsers.add_parser("fmt", help="Format file")
    parser_fmt.add_argument(
        "infile", nargs="?", help="Path for dn42 data file", type=str
    )
    parser_fmt.add_argument(
        "-i", "--in-place", help="Format file in place", action="store_true"
    )

    parser_sane = subparsers.add_parser(
        "sanity-check", help="Check the file for sane-ness"
    )
    parser_sane.add_argument(
        "infile", nargs="?", help="Path for dn42 data file", type=str
    )

    parser_pol = subparsers.add_parser("policy", help="Format file")
    parser_pol.add_argument("type", nargs="?", type=str, help="dn42 object type")
    parser_pol.add_argument("name", nargs="?", type=str, help="dn42 object name")
    parser_pol.add_argument("mntner", nargs="?", type=str, help="dn42 object mntner")
    parser_pol.add_argument("commit", nargs="?", type=str, help="dn42 registry (upstream) commit", default=REGISTRY_COMMIT)

    parser_mroute = subparsers.add_parser(
        "match-routes", help="Match routes to inetnums"
    )
    _ = parser_mroute

    return vars(parser.parse_args())


def run(args):
    "run"
    if args["merge_output"]:
        log.OUTPUT = sys.stdout

    if args["doubleVerbose"]:
        log.default.level_console = log.VERB_DEBUG
        log.default.level_full = True

    if args["verbose"]:
        log.default.level_console = log.VERB_INFO

    log.debug(args)

    valid = True
    if args["command"] == "check-file":
        valid = main(args["infile"], args["use_schema"])
        if valid:
            log.notice("Check %s: PASS" % (args["infile"]))
        else:
            log.fatal("Check %s: FAIL" % (args["infile"]))

    elif args["command"] == "check-schemas":
        valid = check_schemas(args["path"])

    elif args["command"] == "index":
        index_files(args["path"])

    elif args["command"] == "scan-index":
        scan_index(args["infile"], args["use_mntner"])

    elif args["command"] == "scan":
        log.notice(
            "## Scan Started at %s"
            % (time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))
        )
        ck = scan_files(args["path"], args["use_mntner"], args["use_file"])
        log.notice(
            "## Scan Completed at %s"
            % (time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))
        )

        if ck == "INFO":
            sys.exit(2)
        elif ck == "FAIL":
            sys.exit(1)

    elif args["command"] == "fmt":
        dom = FileDOM(args["infile"])
        if args["in_place"]:
            with open(args["infile"], "w+") as f:
                f.write(str(dom))
        else:
            print(str(dom))

    elif args["command"] == "policy":
        
        # check if the commit to check against is old
        now = int(time.time())
        try:
            commit_time = int(_run_command(f"git show -s --date=unix --format=%cd {args['commit']}".split(" ")).strip()) # should return only the unix timestamp of the commit
        except ValueError as e:
            log.fatal(f"could not determine time of the provided commit: {args['commit']}")
        
        if now - commit_time > 60*60*24*14: # more than two weeks(14 days)
            log.warning(f"the commit to check against is older than 14 days, consider fetching/using a newer commit")


        if args["type"] is None:
            log.fatal("Type should be provided")

        if args["name"] is None:
            log.fatal("Name should be provided")

        if args["mntner"] is None:
            log.fatal("Mntner should be provided")

        if args["type"] in ["inetnum", "inet6num", "route", "route6"]:
            args["name"] = args["name"].replace("_", "/")

        status = test_policy(args["type"], args["name"], args["mntner"], commit=args["commit"])

        print(
            "POLICY %-12s\t%-8s\t%20s\t%s"
            % (args["mntner"], args["type"], args["name"], status)
        )
        if status != "PASS":
            sys.exit(1)

    elif args["command"] == "sanity-check":
        dom = FileDOM(args["infile"])
        ck = sanity_check(dom)
        print("SANITY %-8s\t%20s\t%s" % (dom.schema.split(".")[1], args["infile"], ck))
        if ck != "PASS":
            sys.exit(1)

    elif args["command"] == "match-routes":
        # TODO: implement returning @netlevel fields in find_new
        lis = find_old(
            ["mnt-by", "cidr", "route", "@netlevel", "@netmin", "@netmax", "@uri"],
            {"@family": "ipv4"},
        )

        def field(x, field):
            for i in x:
                if i[0] == field:
                    return i[1]
            return None

        def lvl(x):
            for i in x:
                if i[0] == "@netlevel":
                    return i[1]

        def net(x):
            for i in x:
                if i[0] == "@netmin":
                    return i[1]

        def is_net(x):
            i = field(x, "cidr")
            if i is not None:
                return True
            return False

        def obj(x):
            d = {}
            for k, v in x:
                if k in d:
                    d[k].append(v)
                else:
                    d[k] = [v]
            return d

        inet = None
        first = True
        for n in sorted(sorted(lis, key=lvl), key=net):
            o = obj(n)

            if is_net(n):
                if not first:
                    print()
                first = True
                inet = o
                continue

            ilvl = int(inet["@netlevel"][0])
            rlvl = int(o["@netlevel"][0])

            if ilvl + 1 != rlvl:
                print(
                    "\nNo Parent > ",
                    o["route"][0],
                    " ",
                    rlvl,
                    " ",
                    ",".join(o["mnt-by"]),
                    "Nearest INET ",
                    inet["cidr"][0],
                    " ",
                    ilvl,
                    " ",
                    ",".join(inet["mnt-by"]),
                )

                first = True
                continue

            if inet["@netmin"][0] > o["@netmin"][0] or inet["@netmax"][0] < o["@netmax"][0]:
                print(
                    "\nNo Parent > ",
                    o["route"][0],
                    " ",
                    rlvl,
                    " ",
                    ",".join(o["mnt-by"]),
                    "Nearest INET ",
                    inet["cidr"][0],
                    " ",
                    ilvl,
                    " ",
                    ",".join(inet["mnt-by"]),
                )

                first = True
                continue


if __name__ == "__main__":
    run(get_args())
