diff --git a/.activate b/.activate new file mode 100644 index 0000000..a3607ff --- /dev/null +++ b/.activate @@ -0,0 +1,7 @@ + +alias venv='source .venv/Scripts/activate' +alias check='flake8 vpn_manager' +alias format='black vpn_manager' +alias test='pytest' + +venv diff --git a/.gitignore b/.gitignore index 09875ae..11ba77a 100644 --- a/.gitignore +++ b/.gitignore @@ -162,4 +162,4 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ diff --git a/README.md b/README.md index c5f97b1..9e19b31 100644 --- a/README.md +++ b/README.md @@ -26,4 +26,4 @@ run # script declared in pyproject.toml $ example run -``` \ No newline at end of file +``` diff --git a/archive/_test_cidr4_merger.py b/archive/_test_cidr4_merger.py new file mode 100644 index 0000000..c891934 --- /dev/null +++ b/archive/_test_cidr4_merger.py @@ -0,0 +1,395 @@ +import pytest + +from cidr4_merger import ( + Cidr4MergerError, + cidr4_to_node, + find_neighbours_singles, + get_group_with_max_mask_len, + get_net_addr, + get_parent_ip, + have_same_parent, + lift_lonely_node, + make_cidr4, + make_groups, + make_parent, + merge_neighbors, + merge_nodes_cycle, + merge_nodes_deprecated, + merge_nodes_recursion, + reduce_nodes, + sort_nodes, +) + +from .cidr4_data_for_tests import test_cidr4_data + + +def test_true(): + assert True + + +bin_a = "10011000000000001000010000010000" +assert len(bin_a) == 32 +ip_a = int(bin_a, 2) + +bin_b = "10011100000000000000000000101011" +assert len(bin_b) == 32 +ip_b = int(bin_b, 2) + +bin_c = "10011000000000000000000000000000" +assert len(bin_c) == 32 +ip_c = int(bin_c, 2) + +bin_d = "11111100000000000000000000000000" +assert len(bin_c) == 32 +ip_d = int(bin_d, 2) + + +def test_cidr4_to_node(): + assert cidr4_to_node("4.78.139.0/24") == (72256256, 24, 0, 72256000) + assert cidr4_to_node("0.0.0.0/32") == (0, 32, 0, 0) + + assert cidr4_to_node("23.234.30.0/24") == (401219072, 24, 0, 401219072) + assert cidr4_to_node("172.217.0.0/19") == (2899902464, 19, 0, 2899902464) + assert cidr4_to_node("23.225.141.0/24") == (400657664, 24, 0, 400657408) + assert cidr4_to_node("31.13.94.0/23") == (520969728, 23, 0, 520969216) + + assert cidr4_to_node("0.0.0.0/2") == (0, 2, 0, 0) + assert cidr4_to_node("64.0.0.0/2") == (1073741824, 2, 0, 0) + assert cidr4_to_node("128.0.0.0/2") == (2147483648, 2, 0, 2147483648) + assert cidr4_to_node("192.0.0.0/2") == (3221225472, 2, 0, 2147483648) + + +def test_make_cidr4(): + assert make_cidr4(72256256, 24) == "4.78.139.0/24" + assert make_cidr4(0, 32) == "0.0.0.0/32" + + assert make_cidr4(401219072, 24) == "23.234.30.0/24" + assert make_cidr4(2899902464, 19) == "172.217.0.0/19" + assert make_cidr4(400657664, 24) == "23.225.141.0/24" + assert make_cidr4(520969728, 23) == "31.13.94.0/23" + + +def test_get_net_addr(): + assert get_net_addr(ip_a, 5) == ip_c + assert get_net_addr(ip_b, 5) == ip_c + assert get_net_addr(0, 1) == 0 + assert get_net_addr(0, 0) == 0 + + +def test_get_parent_mask(): + assert get_parent_ip(ip_a, 6) == ip_c + assert get_parent_ip(ip_b, 6) == ip_c + assert get_parent_ip(0, 1) == 0 + + with pytest.raises(Exception) as exc_info: + get_parent_ip(0, 0) + assert str(exc_info.value) == "The top of the tree has no parent!" + assert exc_info.type is Cidr4MergerError + + +def test_have_same_parent(): + assert have_same_parent(ip_c, 6, ip_c, 6) is True + assert have_same_parent(ip_c, 6, ip_c, 5) is False + assert have_same_parent(ip_a, 6, ip_d, 6) is False + assert have_same_parent(ip_a, 6, 0, 1) is False + assert have_same_parent(ip_a, 6, 0, 0) is False + + +def test_sort_nodes(): + assert sort_nodes( + [ + (401219072, 24, 0, 401219072), + (2899902464, 19, 0, 2899902464), + (400657664, 24, 0, 400657408), + (520969728, 23, 0, 520969216), + ] + ) == [ + (2899902464, 19, 0, 2899902464), + (520969728, 23, 0, 520969216), + (400657664, 24, 0, 400657408), + (401219072, 24, 0, 401219072), + ] + + +def test_get_group_with_max_mask_len(): + assert get_group_with_max_mask_len( + [ + (2899902464, 19, 0, 2899902464), + (520969728, 23, 0, 520969216), + (400657664, 24, 0, 400657408), + (401219072, 24, 0, 401219072), + ] + ) == [(400657664, 24, 0, 400657408), (401219072, 24, 0, 401219072)] + + assert get_group_with_max_mask_len( + [ + (401219072, 24, 0, 401219072), + (2899902464, 19, 0, 2899902464), + (520969728, 23, 0, 520969216), + ] + ) == [(401219072, 24, 0, 401219072)] + + +def test_make_parent(): + assert make_parent((0, 2, 12, 0), (1073741824, 2, 3, 0)) == (0, 1, 15, 0) + assert make_parent( + (2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648) + ) == (2147483648, 1, 3, 0) + + with pytest.raises(Exception) as exc_info: + make_parent((0, 2, 12, 0), (3221225472, 2, 2, 2147483648)) + assert str(exc_info.value) == "Nodes must be neighbors!" + assert exc_info.type is Cidr4MergerError + + +def test_reduce_nodes(): + assert reduce_nodes( + [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + ] + ) == [ + (0, 1, 15, 0), + ] + + assert reduce_nodes( + [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ] + ) == [ + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + ] + + assert reduce_nodes( + [ + (0, 2, 12, 0), + (2147483648, 1, 0, 0), + ] + ) == [ + (0, 1, 12 + 2**30, 0), + (2147483648, 1, 0, 0), + ] + + with pytest.raises(Exception) as exc_info: + reduce_nodes( + [ + (0, 1, 12 + 2**30, 0), + (2147483648, 1, 0, 0), + ] + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_merge_nodes_deprecated(): + assert merge_nodes_deprecated( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 2, + ) == [ + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + ] + + with pytest.raises(Exception) as exc_info: + merge_nodes_deprecated( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 1, + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_make_groups(): + nodes = [ + (2398793728, 20, 0, 2398789632), + (2899943424, 20, 0, 2899943424), + (3627728896, 20, 0, 3627728896), + (520963072, 22, 0, 520962048), + (1089054720, 22, 0, 1089054720), + (2899902464, 19, 0, 2899902464), + (2915221504, 19, 0, 2915221504), + ] + assert dict(make_groups(nodes)) == { + 20: [ + (2398793728, 20, 0, 2398789632), + (2899943424, 20, 0, 2899943424), + (3627728896, 20, 0, 3627728896), + ], + 22: [(520963072, 22, 0, 520962048), (1089054720, 22, 0, 1089054720)], + 19: [(2899902464, 19, 0, 2899902464), (2915221504, 19, 0, 2915221504)], + } + + +@pytest.fixture +def nodes_only_neighbours(): + return [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ] + + +@pytest.fixture +def nodes_only_singles(): + return [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + ] + + +@pytest.fixture +def nodes_with_neighbours_n_singles(): + return [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + (2147483648, 2, 1, 2147483648), + ] + + +@pytest.fixture +def groups_only_neighbours(nodes_only_neighbours): + return make_groups(nodes_only_neighbours) + + +@pytest.fixture +def groups_only_singles(nodes_only_singles): + return make_groups(nodes_only_singles) + + +@pytest.fixture +def groups_with_neighbours_n_singles(nodes_with_neighbours_n_singles): + return make_groups(nodes_with_neighbours_n_singles) + + +def test_find_neighbours_singles( + groups_only_neighbours, + groups_only_singles, + groups_with_neighbours_n_singles, +): + assert find_neighbours_singles(groups_only_neighbours) == ( + [ + ((0, 2, 12, 0), (1073741824, 2, 3, 0)), + ((2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648)), + ], + [], + ) + + assert find_neighbours_singles(groups_only_singles) == ( + [], + [(0, 2, 12, 0), (2147483648, 2, 1, 2147483648)], + ) + + assert find_neighbours_singles(groups_with_neighbours_n_singles) == ( + [((0, 2, 12, 0), (1073741824, 2, 3, 0))], + [(2147483648, 2, 1, 2147483648)], + ) + + +def test_merge_neighbors__only_neighbours(nodes_only_neighbours): + new_nodes = merge_neighbors( + nodes_only_neighbours, + [ + ((0, 2, 12, 0), (1073741824, 2, 3, 0)), + ((2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648)), + ], + ) + assert new_nodes == [ + (0, 1, 15, 0), + (2147483648, 1, 3, 0), + ] + + +def test_merge_neighbors__neighbours_n_singles(nodes_with_neighbours_n_singles): + new_nodes = merge_neighbors( + nodes_with_neighbours_n_singles, + [((0, 2, 12, 0), (1073741824, 2, 3, 0))], + ) + assert new_nodes == [ + (0, 1, 15, 0), + (2147483648, 2, 1, 2147483648), + ] + + +def test_lift_lonely_node(nodes_only_singles): + singles = [(0, 2, 12, 0), (2147483648, 2, 1, 2147483648)] + new_nodes = lift_lonely_node(nodes_only_singles, singles) + assert new_nodes == [(2147483648, 1, 1073741825, 0), (0, 2, 12, 0)] + + +def test_merge_nodes_recursion(): + assert merge_nodes_recursion( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 2, + ) == [ + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + ] + + with pytest.raises(Exception) as exc_info: + merge_nodes_recursion( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 1, + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_merge_nodes_cycle(): + assert merge_nodes_cycle( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 2, + ) == [ + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + ] + + with pytest.raises(Exception) as exc_info: + merge_nodes_cycle( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 1, + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_merge_nodes_recursion_vs_cycle(): + required_len = 20 + + test_data = test_cidr4_data.strip().splitlines() + test_nodes = list(map(cidr4_to_node, test_data)) + test_nodes = sort_nodes(test_nodes) + + merged_nodes_recursion = merge_nodes_recursion(test_nodes, required_len) + merged_nodes_cycle = merge_nodes_cycle(test_nodes, required_len) + + assert merged_nodes_recursion == merged_nodes_cycle diff --git a/archive/cidr4_merger.py b/archive/cidr4_merger.py new file mode 100644 index 0000000..7dfa3d0 --- /dev/null +++ b/archive/cidr4_merger.py @@ -0,0 +1,226 @@ +import cProfile +import sys +from collections import defaultdict + +sys.setrecursionlimit(10_000) + +Node = tuple[int, int, int, int] + + +class Cidr4MergerError(Exception): + pass + + +def get_data(input_file): + with open(input_file, "r") as file: + data = file.read().splitlines() + return data + + +def cidr4_to_node(cidr4: str) -> Node: + ip_address, mask_len = cidr4.strip().split("/") + mask_len = int(mask_len) + a, b, c, d = list(map(int, ip_address.split("."))) + ip = a * 256**3 + b * 256**2 + c * 256**1 + d * 256**0 + added_ips_number = 0 + parent_ip = get_parent_ip(ip, mask_len) + return ip, mask_len, added_ips_number, parent_ip + + +def sort_nodes(nodes: list[Node]) -> list[Node]: + return sorted(nodes, key=lambda x: (x[1], x[0])) + + +def get_net_addr(ip: int, mask_len: int) -> int: + mask = ((1 << mask_len) - 1) << (32 - mask_len) + net_addr = ip & mask + return net_addr + + +def get_parent_ip(ip: int, mask_len: int) -> int: + if mask_len == 0: + raise Cidr4MergerError("The top of the tree has no parent!") + return get_net_addr(ip, mask_len - 1) + + +def have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b) -> bool: + return mask_len_a == mask_len_b and parent_ip_a == parent_ip_b + + +def get_group_with_max_mask_len(nodes: list[Node]) -> list[Node]: + max_mask_len = max(nodes, key=lambda x: x[1])[1] + return list(filter(lambda x: x[1] == max_mask_len, nodes)) + + +def make_parent(a: Node, b: Node | None = None) -> Node: + ip_a, mask_len_a, added_ips_number_a, parent_ip_a = a + if b: + ip_b, mask_len_b, added_ips_b, parent_ip_b = b + if not have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): + raise Cidr4MergerError("Nodes must be neighbors!") + added_ips_number = added_ips_number_a + added_ips_b + else: + added_ips_number = added_ips_number_a + 2 ** (32 - mask_len_a) + ip = parent_ip_a + mask_len = mask_len_a - 1 + parent_ip = get_parent_ip(ip, mask_len) + return ip, mask_len, added_ips_number, parent_ip + + +def reduce_nodes(nodes: list[Node]) -> list[Node]: + group = get_group_with_max_mask_len(nodes) + + neighbours = [] + loners = [] + i = 0 + while i < len(group) - 1: + a, b = group[i], group[i + 1] + ip_a, mask_len_a, _, parent_ip_a = a + ip_b, mask_len_b, _, parent_ip_b = b + if have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): + neighbours.append((a, b)) + i += 2 + else: + loners.append(a) + i += 1 + if i == len(group) - 1: + loners.append(group[i]) + + if neighbours: + zipped = zip(neighbours, map(lambda x: make_parent(x[0], x[1]), neighbours)) + min_zipped = min(zipped, key=lambda x: x[1][2]) + (a, b), parent = min_zipped + nodes.remove(a) + nodes.remove(b) + nodes.append(parent) + elif loners: + zipped = zip(loners, map(make_parent, loners)) + min_zipped = min(zipped, key=lambda x: x[1][2]) + a, parent = min_zipped + nodes.remove(a) + nodes.append(parent) + else: + assert False, "Error" + + return sort_nodes(nodes) + + +def merge_nodes_deprecated(nodes: list[Node], required_len: int) -> list[Node]: + while len(nodes) > required_len: + nodes = reduce_nodes(nodes) + return nodes + + +def make_cidr4(ip, mask_len) -> str: + lst = [str(ip >> (i << 3) & 0xFF) for i in reversed(range(4))] + ip_address = ".".join(lst) + return f"{ip_address}/{mask_len}" + + +def lift_lonely_node(nodes: list[Node], singles: list[Node]) -> list[Node]: + # find single whose parent has the least added addresses + min_single, min_parent = singles[0], make_parent(singles[0]) + for node in singles[1:]: + parent = make_parent(node) + if parent[2] < min_parent[2]: + min_single, min_parent = node, parent + + nodes.remove(min_single) + nodes.append(min_parent) + nodes = sort_nodes(nodes) + return nodes + + +def merge_neighbors( + nodes: list[Node], neighbours: list[tuple[Node, Node]] +) -> list[Node]: + for a, b in neighbours: + parent = make_parent(a, b) + nodes.remove(a) + nodes.remove(b) + nodes.append(parent) + return sort_nodes(nodes) + + +def find_neighbours_singles(groups: defaultdict) -> tuple[list, list]: + neighbours = [] + singles = [] + for group in groups.values(): + i = 0 + while i < len(group) - 1: + a, b = group[i], group[i + 1] + ip_a, mask_len_a, _, parent_ip_a = a + ip_b, mask_len_b, _, parent_ip_b = b + if have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): + neighbours.append((a, b)) + i += 2 + else: + singles.append(a) + i += 1 + if i == len(group) - 1: + singles.append(group[i]) + return neighbours, singles + + +def make_groups(nodes: list[Node]) -> defaultdict: + groups = defaultdict(list) + for n in nodes: + groups[n[1]].append(n) + return groups + + +def merge_nodes_recursion(nodes: list[Node], required_len: int) -> list[Node]: + if len(nodes) <= required_len: + return nodes + groups = make_groups(nodes) + neighbours, singles = find_neighbours_singles(groups) + if neighbours: + new_nodes = merge_neighbors(nodes, neighbours) + return merge_nodes_recursion(new_nodes, required_len) + new_nodes = lift_lonely_node(nodes, singles) + return merge_nodes_recursion(new_nodes, required_len) + + +def merge_nodes_cycle(nodes_to_merge: list[Node], required_len: int) -> list[Node]: + nodes = [x for x in nodes_to_merge] + while not len(nodes) <= required_len: + groups = make_groups(nodes) + neighbours, singles = find_neighbours_singles(groups) + if neighbours: + nodes = merge_neighbors(nodes, neighbours) + elif singles: + nodes = lift_lonely_node(nodes, singles) + else: + raise Cidr4MergerError("Invalid case!") + return nodes + + +def main(): + file = "cidr4.txt" + required_len = 20 + + data = get_data(file) + nodes = list(map(cidr4_to_node, data)) + + nodes = sort_nodes(nodes) + # merged_nodes = merge_nodes_deprecated(nodes, required_len) + # merged_nodes = merge_nodes_recursion(nodes, required_len) + merged_nodes = merge_nodes_cycle(nodes, required_len) + + cidr4s = [] + sum_added_ips = 0 + for ip_value, mask_len, added_ips, _ in merged_nodes: + cidr4s.append(make_cidr4(ip_value, mask_len)) + sum_added_ips += added_ips + + cidr4s_str = "\n".join(cidr4s) + print( + f"Исходный список длины {len(nodes)} сокращен до {len(cidr4s)}\n" + f"Количество добавленных ip адресов: {sum_added_ips:_}\n" + f"Список объединенных cidr4:\n" + f"{cidr4s_str}" + ) + + +if __name__ == "__main__": + cProfile.run("main()") diff --git a/archive/insurance_functions_example.md b/archive/insurance_functions_example.md new file mode 100644 index 0000000..8b1a256 --- /dev/null +++ b/archive/insurance_functions_example.md @@ -0,0 +1,30 @@ +```python +def find_subnets(nodes: list[Node]) -> list[tuple[Node, Node]]: + subnets = [] + for i, (a, b) in enumerate(zip(nodes, nodes[1:])): + parent_node, dip = merge_two_nodes(a, b) + if parent_node == a or parent_node == b: + subnets.append((a, b)) + return subnets + + +def ensure_no_subnets(nodes: list[Node]): + if subnets := find_subnets(nodes): + raise Cidr4MergerError(f"There are subnets! {subnets=}") +``` + + +```python +def find_neighbors(nodes: list[Node]) -> list[tuple[Node, Node]]: + neighbors = [] + for i, (a, b) in enumerate(zip(nodes, nodes[1:])): + parent_node, dip = merge_two_nodes(a, b) + if parent_node[1] + 1 == a[1] == b[1]: + neighbors.append((a, b)) + return neighbors + + +def ensure_no_neighbors(nodes: list[Node]): + if neighbors := find_neighbors(nodes): + raise Cidr4MergerError(f"There are neighbors! {neighbors=}") +``` diff --git a/cidr4.txt b/cidr4.txt new file mode 100644 index 0000000..b3f117f --- /dev/null +++ b/cidr4.txt @@ -0,0 +1,486 @@ +4.78.139.0/24 +23.101.24.0/24 +23.202.231.0/24 +23.217.138.0/24 +23.225.141.0/24 +23.234.30.0/24 +31.13.64.0/24 +31.13.67.0/24 +31.13.68.0/22 +31.13.73.0/24 +31.13.75.0/24 +31.13.76.0/24 +31.13.80.0/21 +31.13.88.0/24 +31.13.90.0/23 +31.13.92.0/24 +31.13.94.0/23 +31.13.96.0/24 +31.13.106.0/24 +31.13.112.0/24 +37.152.2.0/24 +38.121.72.0/24 +39.109.122.0/24 +43.226.16.0/24 +43.245.104.0/24 +45.54.28.0/24 +45.77.186.0/24 +45.114.11.0/24 +45.253.131.0/24 +46.61.154.0/24 +46.134.216.0/24 +47.88.58.0/24 +49.231.55.0/24 +50.23.209.0/24 +50.87.93.0/24 +50.117.117.0/24 +52.58.1.0/24 +52.175.9.0/24 +54.89.135.0/24 +54.144.128.0/24 +54.234.18.0/24 +58.63.233.0/24 +59.18.44.0/24 +59.18.46.0/24 +59.24.3.0/24 +59.188.250.0/24 +61.91.8.0/24 +61.205.119.0/24 +62.0.80.0/24 +64.13.192.0/24 +64.53.242.0/24 +64.233.160.0/21 +64.233.168.0/22 +64.233.176.0/20 +65.49.26.0/24 +65.49.68.0/24 +66.102.1.0/24 +66.220.146.0/23 +66.220.148.0/23 +67.15.100.0/24 +67.15.129.0/24 +67.228.102.0/24 +67.228.235.0/24 +67.230.169.0/24 +69.30.25.0/24 +69.50.221.0/24 +69.63.176.0/24 +69.63.178.0/24 +69.63.180.0/23 +69.63.184.0/24 +69.63.186.0/23 +69.63.190.0/24 +69.162.134.0/24 +69.171.224.0/24 +69.171.227.0/24 +69.171.228.0/23 +69.171.234.0/24 +69.171.242.0/24 +69.171.247.0/24 +69.197.153.0/24 +74.86.3.0/24 +74.86.12.0/24 +74.86.17.0/24 +74.86.118.0/24 +74.86.142.0/24 +74.86.151.0/24 +74.86.226.0/24 +74.86.228.0/24 +74.125.1.0/24 +74.125.2.0/24 +74.125.8.0/24 +74.125.10.0/23 +74.125.13.0/24 +74.125.20.0/23 +74.125.23.0/24 +74.125.24.0/24 +74.125.26.0/24 +74.125.28.0/24 +74.125.31.0/24 +74.125.68.0/22 +74.125.90.0/24 +74.125.96.0/24 +74.125.100.0/24 +74.125.102.0/23 +74.125.104.0/24 +74.125.106.0/24 +74.125.110.0/23 +74.125.124.0/24 +74.125.126.0/23 +74.125.128.0/20 +74.125.153.0/24 +74.125.154.0/23 +74.125.159.0/24 +74.125.164.0/24 +74.125.170.0/24 +74.125.172.0/23 +74.125.192.0/23 +74.125.195.0/24 +74.125.196.0/22 +74.125.200.0/22 +74.125.204.0/23 +74.125.206.0/24 +75.126.33.0/24 +75.126.115.0/24 +75.126.124.0/24 +75.126.135.0/24 +75.126.150.0/24 +75.126.164.0/24 +77.37.252.0/24 +77.120.15.0/24 +80.87.199.0/24 +81.23.20.0/24 +81.23.23.0/24 +81.192.13.0/24 +81.192.191.0/24 +88.191.249.0/24 +90.201.124.0/24 +92.87.232.0/24 +93.179.102.0/24 +94.24.232.0/24 +94.31.189.0/24 +95.59.170.0/24 +96.44.137.0/24 +98.159.108.0/24 +103.39.76.0/24 +103.42.176.0/24 +103.56.16.0/24 +103.73.161.0/24 +103.97.3.0/24 +103.97.176.0/24 +103.200.30.0/23 +103.214.168.0/24 +103.226.246.0/24 +103.228.130.0/24 +103.230.123.0/24 +103.240.180.0/24 +103.240.182.0/24 +103.246.246.0/24 +103.252.114.0/23 +104.16.251.0/24 +104.16.252.0/24 +104.23.124.0/23 +104.31.142.0/24 +104.244.43.0/24 +104.244.45.0/24 +104.244.46.0/24 +107.181.166.0/24 +108.160.161.0/24 +108.160.162.0/23 +108.160.165.0/24 +108.160.166.0/23 +108.160.169.0/24 +108.160.170.0/24 +108.160.172.0/23 +108.177.8.0/21 +108.177.96.0/23 +108.177.98.0/24 +108.177.103.0/24 +108.177.104.0/24 +108.177.111.0/24 +108.177.112.0/24 +108.177.119.0/24 +108.177.120.0/22 +108.177.125.0/24 +108.177.126.0/23 +109.224.41.0/24 +110.164.8.0/24 +111.243.214.0/24 +113.108.239.0/24 +113.171.242.0/24 +114.4.7.0/24 +114.43.24.0/24 +115.126.100.0/24 +116.89.243.0/24 +118.98.30.0/24 +118.98.36.0/24 +118.98.106.0/24 +118.107.180.0/24 +118.184.26.0/24 +118.184.78.0/24 +118.193.202.0/24 +118.193.240.0/24 +119.28.87.0/24 +120.232.233.0/24 +120.232.234.0/24 +120.233.71.0/24 +120.253.250.0/24 +120.253.253.0/24 +120.253.255.0/24 +121.78.42.0/24 +122.10.85.0/24 +122.154.76.0/24 +122.248.226.0/24 +122.252.245.0/24 +124.11.210.0/24 +128.121.146.0/24 +128.121.243.0/24 +128.242.240.0/24 +128.242.245.0/24 +128.242.250.0/24 +130.211.15.0/24 +142.250.0.0/23 +142.250.4.0/24 +142.250.8.0/22 +142.250.12.0/23 +142.250.27.0/24 +142.250.28.0/24 +142.250.30.0/23 +142.250.64.0/20 +142.250.80.0/23 +142.250.96.0/21 +142.250.105.0/24 +142.250.107.0/24 +142.250.110.0/23 +142.250.112.0/22 +142.250.123.0/24 +142.250.125.0/24 +142.250.126.0/24 +142.250.128.0/24 +142.250.136.0/24 +142.250.138.0/24 +142.250.141.0/24 +142.250.142.0/24 +142.250.145.0/24 +142.250.147.0/24 +142.250.148.0/23 +142.250.150.0/24 +142.250.152.0/23 +142.250.157.0/24 +142.250.158.0/23 +142.250.176.0/20 +142.250.192.0/21 +142.250.200.0/23 +142.250.203.0/24 +142.250.204.0/22 +142.250.217.0/24 +142.250.218.0/23 +142.251.0.0/23 +142.251.2.0/24 +142.251.4.0/23 +142.251.6.0/24 +142.251.8.0/23 +142.251.10.0/24 +142.251.12.0/24 +142.251.15.0/24 +142.251.16.0/24 +142.251.18.0/24 +142.251.31.0/24 +142.251.32.0/22 +142.251.36.0/23 +142.251.39.0/24 +142.251.40.0/22 +142.251.45.0/24 +142.251.46.0/23 +142.251.107.0/24 +142.251.111.0/24 +142.251.112.0/24 +142.251.116.0/23 +142.251.120.0/24 +142.251.128.0/23 +142.251.130.0/24 +142.251.132.0/22 +142.251.140.0/22 +142.251.161.0/24 +142.251.162.0/23 +142.251.164.0/22 +142.251.168.0/24 +142.251.170.0/23 +142.251.172.0/22 +142.251.176.0/22 +142.251.180.0/24 +142.251.182.0/23 +142.251.184.0/22 +142.251.188.0/24 +142.251.208.0/23 +142.251.211.0/24 +142.251.214.0/23 +142.251.216.0/24 +142.251.218.0/24 +142.251.220.0/22 +145.255.14.0/24 +148.163.48.0/24 +150.107.3.0/24 +154.0.29.0/24 +154.83.14.0/23 +154.85.102.0/24 +154.92.16.0/24 +156.233.67.0/24 +157.240.0.0/22 +157.240.6.0/23 +157.240.8.0/22 +157.240.12.0/23 +157.240.15.0/24 +157.240.16.0/23 +157.240.18.0/24 +157.240.20.0/23 +159.65.107.0/24 +159.106.121.0/24 +159.138.20.0/24 +162.125.1.0/24 +162.125.2.0/24 +162.125.6.0/23 +162.125.8.0/24 +162.125.17.0/24 +162.125.18.0/24 +162.125.32.0/24 +162.125.34.0/24 +162.125.80.0/24 +162.125.82.0/23 +162.220.12.0/24 +168.143.162.0/24 +168.143.171.0/24 +172.217.0.0/19 +172.217.129.0/24 +172.217.130.0/24 +172.217.133.0/24 +172.217.135.0/24 +172.217.160.0/20 +172.217.192.0/22 +172.217.197.0/24 +172.217.203.0/24 +172.217.204.0/24 +172.217.212.0/24 +172.217.214.0/23 +172.217.218.0/23 +172.217.222.0/24 +172.253.58.0/24 +172.253.62.0/23 +172.253.112.0/21 +172.253.120.0/24 +172.253.122.0/23 +172.253.124.0/22 +173.194.4.0/23 +173.194.12.0/24 +173.194.22.0/24 +173.194.28.0/23 +173.194.31.0/24 +173.194.49.0/24 +173.194.51.0/24 +173.194.54.0/23 +173.194.59.0/24 +173.194.65.0/24 +173.194.66.0/23 +173.194.68.0/23 +173.194.70.0/24 +173.194.73.0/24 +173.194.74.0/24 +173.194.76.0/22 +173.194.135.0/24 +173.194.150.0/24 +173.194.154.0/24 +173.194.161.0/24 +173.194.162.0/23 +173.194.164.0/24 +173.194.166.0/23 +173.194.174.0/23 +173.194.178.0/24 +173.194.182.0/23 +173.194.184.0/23 +173.194.187.0/24 +173.194.188.0/24 +173.194.190.0/23 +173.194.192.0/19 +173.208.182.0/24 +173.231.12.0/24 +173.234.53.0/24 +173.236.182.0/24 +173.236.212.0/24 +173.244.209.0/24 +173.244.217.0/24 +173.252.88.0/24 +173.252.105.0/24 +173.252.108.0/24 +173.252.248.0/24 +173.255.209.0/24 +173.255.213.0/24 +174.36.196.0/24 +174.36.228.0/24 +174.37.54.0/24 +174.37.154.0/24 +174.37.175.0/24 +174.37.243.0/24 +178.151.230.0/24 +178.176.156.0/24 +179.60.193.0/24 +180.163.150.0/23 +182.50.139.0/24 +182.79.251.0/24 +184.72.1.0/24 +184.173.136.0/24 +185.45.6.0/23 +185.60.216.0/24 +185.60.218.0/23 +185.158.208.0/24 +186.208.210.0/24 +187.7.116.0/24 +190.5.235.0/24 +192.133.77.0/24 +192.178.24.0/23 +192.178.27.0/24 +192.178.48.0/23 +192.178.50.0/24 +192.178.52.0/24 +192.178.54.0/24 +192.178.56.0/23 +192.178.128.0/24 +192.178.130.0/24 +193.109.164.0/24 +194.78.0.0/24 +196.49.8.0/24 +198.27.124.0/24 +198.44.185.0/24 +199.16.156.0/24 +199.16.158.0/24 +199.59.148.0/23 +199.59.150.0/24 +199.96.58.0/23 +199.96.61.0/24 +199.96.62.0/23 +199.193.116.0/24 +201.0.223.0/24 +202.53.137.0/24 +202.160.128.0/23 +202.160.130.0/24 +202.169.173.0/24 +202.182.98.0/24 +203.66.182.0/24 +203.111.254.0/24 +203.113.51.0/24 +203.113.189.0/24 +203.208.39.0/24 +203.208.40.0/23 +203.208.43.0/24 +203.208.49.0/24 +203.208.50.0/24 +203.233.96.0/24 +204.79.197.0/24 +205.186.152.0/24 +208.31.254.0/24 +208.43.170.0/24 +208.43.237.0/24 +208.77.47.0/24 +208.101.21.0/24 +208.101.60.0/24 +209.85.144.0/22 +209.85.165.0/24 +209.85.200.0/22 +209.85.224.0/24 +209.85.226.0/24 +209.85.232.0/22 +209.95.56.0/24 +210.56.51.0/24 +210.139.253.0/24 +210.209.84.0/24 +211.104.160.0/24 +212.113.52.0/24 +213.59.210.0/24 +216.58.192.0/20 +216.58.208.0/21 +216.58.217.0/24 +216.58.220.0/22 +216.239.32.0/24 +216.239.34.0/24 +216.239.36.0/24 +216.239.38.0/24 +220.181.174.0/24 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cidr4_merge/test_cidr4_merger.py b/tests/cidr4_merge/test_cidr4_merger.py new file mode 100644 index 0000000..6d636db --- /dev/null +++ b/tests/cidr4_merge/test_cidr4_merger.py @@ -0,0 +1,70 @@ +from vpn_manager.cidr4_merge.cidr4_merger import find_neighbors, merge_nodes, solution + + +def test_true(): + assert True + + +def test_merge_nodes(): + assert merge_nodes((0, 32), (1, 32)) == ((0, 31), 0) + assert merge_nodes((0, 32), (2, 32)) == ((0, 30), 2) + assert merge_nodes((0, 32), (5, 32)) == ((0, 29), 6) + assert merge_nodes((3, 32), (4, 32)) == ((0, 29), 6) + assert merge_nodes((0, 32), (4, 30)) == ((0, 29), 3) + assert merge_nodes((0, 32), (6, 31)) == ((0, 29), 5) + + +def test_find_neighbors(): + assert find_neighbors([(0, 32), (2, 32), (4, 32), (6, 32)]) == [] + assert find_neighbors([(0, 32), (1, 32)]) == [(0, (0, 32), (1, 32))] + assert find_neighbors( + [ + (0, 32), + (1, 32), + (6, 32), + (7, 32), + ] + ) == [ + (0, (0, 32), (1, 32)), + (2, (6, 32), (7, 32)), + ] + assert find_neighbors( + [ + (0, 32), + (1, 32), + (6, 32), + ] + ) == [ + (0, (0, 32), (1, 32)), + ] + + +def test_solution(): + assert solution( + [ + (0, 32), + (3, 32), + (4, 32), + ], + 2, + ) == ([(0, 30), (4, 32)], 2) + + assert solution( + [ + (0, 32), + (3, 32), + (4, 32), + (7, 32), + ], + 2, + ) == ([(0, 29)], 4) + + assert solution( + [ + (0, 32), + (3, 32), + (4, 32), + (7, 32), + ], + 1, + ) == ([(0, 29)], 4) diff --git a/tests/cidr4_merge/test_solutions.py b/tests/cidr4_merge/test_solutions.py new file mode 100644 index 0000000..091fb9b --- /dev/null +++ b/tests/cidr4_merge/test_solutions.py @@ -0,0 +1,56 @@ +from vpn_manager.cidr4_merge.fast import solution as fast +from vpn_manager.cidr4_merge.precise import solution as precise, f +from vpn_manager.cidr4_merge.util import * + + +def test_true(): + assert True + +#cidrs = list(map(cidr4_to_node, get_data())) + +def test_fast_single_lifting(): + assert ([(0, 30)], 0) ==\ + fast([(0, 31), (2, 31)], 1) + assert ([(0, 29)], 2) ==\ + fast([(0, 30), (4, 31)], 1) + assert ([(0, 29)], 3) ==\ + fast([(0, 30), (4, 32)], 1) + +def test_fast_double_lifting(): + assert ([(0, 29)], 4) ==\ + fast([(0, 31), (4, 31)], 1) + + +def test_fast_subnets(): + assert ([(0, 30)], 0) ==\ + fast([(0, 30), (0, 31)], 1) + assert ([(0, 29)], 0) ==\ + fast([(0, 29), (4, 31)], 1) + + +def test_precise_single_lifting(): + assert ([(0, 30)], 0) ==\ + precise([(0, 31), (2, 31)], 1) + assert ([(0, 29)], 2) ==\ + precise([(0, 30), (4, 31)], 1) + assert ([(0, 29)], 3) ==\ + precise([(0, 30), (4, 32)], 1) + +def test_precise_double_lifting(): + assert ([(0, 29)], 4) ==\ + precise([(0, 31), (4, 31)], 1) + + +def test_precise_subnets(): + assert ([(0, 30)], 0) ==\ + precise([(0, 30), (0, 31)], 1) + assert ([(0, 29)], 0) ==\ + precise([(0, 29), (4, 31)], 1) + +def test_precise_f_single_lifting(): + assert (0, (0, 30, 0)) ==\ + f((0, 31, 0), (2, 31, 0)) + assert (2, (0, 29, 2)) ==\ + f((0, 30, 0), (4, 31, 0)) + assert (3, (0, 29, 3)) ==\ + f((0, 30, 0), (4, 32, 0)) diff --git a/tests/cidr4_merge/test_util.py b/tests/cidr4_merge/test_util.py new file mode 100644 index 0000000..ea5eeb2 --- /dev/null +++ b/tests/cidr4_merge/test_util.py @@ -0,0 +1,30 @@ +from vpn_manager.cidr4_merge.util import cidr4_to_node, make_cidr4 + + +def test_true(): + assert True + + +def test_cidr4_to_node(): + assert cidr4_to_node("4.78.139.0/24") == (72256256, 24) + assert cidr4_to_node("0.0.0.0/32") == (0, 32) + + assert cidr4_to_node("23.234.30.0/24") == (401219072, 24) + assert cidr4_to_node("172.217.0.0/19") == (2899902464, 19) + assert cidr4_to_node("23.225.141.0/24") == (400657664, 24) + assert cidr4_to_node("31.13.94.0/23") == (520969728, 23) + + assert cidr4_to_node("0.0.0.0/2") == (0, 2) + assert cidr4_to_node("64.0.0.0/2") == (1073741824, 2) + assert cidr4_to_node("128.0.0.0/2") == (2147483648, 2) + assert cidr4_to_node("192.0.0.0/2") == (3221225472, 2) + + +def test_make_cidr4(): + assert make_cidr4(72256256, 24) == "4.78.139.0/24" + assert make_cidr4(0, 32) == "0.0.0.0/32" + + assert make_cidr4(401219072, 24) == "23.234.30.0/24" + assert make_cidr4(2899902464, 19) == "172.217.0.0/19" + assert make_cidr4(400657664, 24) == "23.225.141.0/24" + assert make_cidr4(520969728, 23) == "31.13.94.0/23" diff --git a/vpn_manager/cidr4_merge/README.md b/vpn_manager/cidr4_merge/README.md new file mode 100644 index 0000000..b3c65b0 --- /dev/null +++ b/vpn_manager/cidr4_merge/README.md @@ -0,0 +1,6 @@ +Run merging algorithms: +``` +python -m vpn_manager.cidr4_merge.cidr4_merger +python -m vpn_manager.cidr4_merge.fast +python -m vpn_manager.cidr4_merge.precise +``` diff --git a/vpn_manager/cidr4_merge/__init__.py b/vpn_manager/cidr4_merge/__init__.py new file mode 100644 index 0000000..f8176bd --- /dev/null +++ b/vpn_manager/cidr4_merge/__init__.py @@ -0,0 +1,2 @@ +class Cidr4MergerError(Exception): + pass diff --git a/vpn_manager/cidr4_merge/cidr4_merger.py b/vpn_manager/cidr4_merge/cidr4_merger.py new file mode 100644 index 0000000..683f1f7 --- /dev/null +++ b/vpn_manager/cidr4_merge/cidr4_merger.py @@ -0,0 +1,104 @@ +import cProfile + +from vpn_manager.cidr4_merge import Cidr4MergerError + +from .util import cidr4_to_node, get_data, make_cidr4 + +Node = tuple[int, int] + + +class EnsureNoSubnetError(Cidr4MergerError): + pass + + +class EnsureNoNeighborsError(Cidr4MergerError): + pass + + +def find_subnets(nodes: list[Node]) -> list[tuple[Node, Node]]: + subnets = [] + for i, (a, b) in enumerate(zip(nodes, nodes[1:])): + parent_node, dip = merge_nodes(a, b) + if parent_node == a or parent_node == b: + subnets.append((a, b)) + return subnets + + +def ensure_no_subnets(nodes: list[Node]): + if subnets := find_subnets(nodes): + raise EnsureNoSubnetError(f"There are subnets! {subnets=}") + + +def find_neighbors(nodes: list[Node]) -> list[tuple[int, Node, Node]]: + neighbors = [] + for i, (a, b) in enumerate(zip(nodes, nodes[1:])): + parent_node, dip = merge_nodes(a, b) + if parent_node[1] + 1 == a[1] == b[1]: + neighbors.append((i, a, b)) + return neighbors + + +def ensure_no_neighbors(nodes: list[Node]): + if neighbors := find_neighbors(nodes): + raise EnsureNoNeighborsError(f"There are neighbors! {neighbors=}") + + +def merge_nodes(a: Node, b: Node) -> tuple[Node, int]: + def find_parent(a, b): + ia, la = a + ib, lb = b + min_l = min(la, lb) + mask = ((1 << min_l) - 1) << (32 - min_l) + while ia & mask != ib & mask: + min_l -= 1 + mask = (mask << 1) & ((1 << 32) - 1) + return ia & mask, min_l + + def calc_dip(l1, lp): + m = lp + 1 + res = 1 << (l1 - m) + res -= 1 + res <<= 32 - l1 + return res + + p = find_parent(a, b) + dip = calc_dip(a[1], p[1]) + calc_dip(b[1], p[1]) + return p, dip + + +def solution(nodes: list[Node], req_len: int) -> tuple[list[Node], int]: + sum_dip = 0 + while len(nodes) > req_len: + min_t = None, None, float("inf") + for i, (a, b) in enumerate(zip(nodes, nodes[1:])): + p, dip = merge_nodes(a, b) + if dip < min_t[2]: + min_t = i, p, dip + if dip == 0: + break + i, p, dip = min_t + nodes = nodes[:i] + [p] + nodes[i + 2 :] + sum_dip += dip + + while nbs := find_neighbors(nodes): + i, a, b = nbs[0] + p, dip = merge_nodes(a, b) + nodes = nodes[:i] + [p] + nodes[i + 2 :] + sum_dip += dip + + return nodes, sum_dip + + +def main(): + required_len = 20 + data = get_data() + nodes = sorted(map(cidr4_to_node, data)) + ensure_no_subnets(nodes) + ensure_no_neighbors(nodes) + merged_nodes, sum_dip = solution(nodes, required_len) + cidr4s = [make_cidr4(ip, mask_len) for ip, mask_len in merged_nodes] + print(cidr4s, sum_dip, sep="\n") + + +if __name__ == "__main__": + cProfile.run("main()") diff --git a/vpn_manager/cidr4_merge/fast.py b/vpn_manager/cidr4_merge/fast.py new file mode 100644 index 0000000..ca65b46 --- /dev/null +++ b/vpn_manager/cidr4_merge/fast.py @@ -0,0 +1,43 @@ +from heapq import heapify, heappush, heappop +from .util import mask, get_data, cidr4_to_node, make_cidr4 + + +def solution(cidrs, M): + h = [] + d = {} + for ip, l in cidrs: + h.append((32-l, ip, l)) + d[(ip, l)] = 0 + heapify(h) + while len(h)>M: + x1, ip1, l1 = heappop(h) + x2, ip2, l2 = h[0] + if l1==l2 and ip1 & mask[l1-1] == ip2 & mask[l2-1]: + heappop(h) + if (ip1 & mask[l1-1], l1-1) not in d: + heappush(h, (x1+1, ip1 & mask[l1-1], l1-1)) + d[(ip1 & mask[l1-1], l1-1)] = d[(ip1, l1)] + d[(ip2, l2)] + del d[(ip2, l2)] + else: + if (ip1 & mask[l1-1], l1-1) not in d: + heappush(h, (x1+1, ip1 & mask[l1-1], l1-1)) + d[(ip1 & mask[l1-1], l1-1)] = d[(ip1, l1)] + 2**x1 + del d[(ip1, l1)] + s = sum(d.values()) + cidrs = list(d.keys()) + return cidrs, s + + +def main(): + M = 20 + a = get_data() + b = list(map(cidr4_to_node, a)) + cidrs, s = solution(b, M) + cidrs = sorted([make_cidr4(*x) for x in cidrs]) + print(cidrs, s, sep='\n') + + +if __name__=='__main__': + import cProfile + main() + diff --git a/vpn_manager/cidr4_merge/precise.py b/vpn_manager/cidr4_merge/precise.py new file mode 100644 index 0000000..8dcae41 --- /dev/null +++ b/vpn_manager/cidr4_merge/precise.py @@ -0,0 +1,59 @@ +from .util import mask, get_data, cidr4_to_node, make_cidr4 + + +def f(x, y): + t = x + b = y + if x[1]>y[1]: + t = y + b = x + ip1, l1, a1 = t + ip2, l2, a2 = b + if ip1 & mask[l1] == ip2 & mask[l1]: + return (0, t) + t1 = t2 = 0 + while not l1==l2: + t2 += 2**(32-l2) + l2 -= 1 + ip2 = ip2 & mask[l2] + while not ip1 & mask[l1-1] == ip2 & mask[l2-1]: + t1 += 2**(32-l1) + l1 -= 1 + ip1 = ip1 & mask[l1] + t2 += 2**(32-l2) + l2 -= 1 + ip2 = ip2 & mask[l2] + r = (ip1 & mask[l1-1], l1-1, a1+a2+t1+t2) + return (t1+t2, r) + + +def solution(cidrs, M): + cidrs = sorted((ip, l, 0) for ip, l in cidrs) + while len(cidrs)>M: + t = (None, float('+inf'), None) + for i, (x, y) in enumerate(zip(cidrs, cidrs[1:])): + m, r = f(x, y) + if m str: + lst = [str(ip >> (i << 3) & 0xFF) for i in reversed(range(4))] + ip_address = ".".join(lst) + return f"{ip_address}/{mask_len}"