From 2cab2c31a6f40dcc315d19e82463a20934a95b8a Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Wed, 22 Jan 2025 17:49:20 +0300 Subject: [PATCH 1/3] add current solution to archive before changes --- archive/_test_cidr4_merger.py | 395 ++++++++++++++++++++++++++++++++++ archive/cidr4_merger.py | 226 +++++++++++++++++++ 2 files changed, 621 insertions(+) create mode 100644 archive/_test_cidr4_merger.py create mode 100644 archive/cidr4_merger.py diff --git a/archive/_test_cidr4_merger.py b/archive/_test_cidr4_merger.py new file mode 100644 index 0000000..c891934 --- /dev/null +++ b/archive/_test_cidr4_merger.py @@ -0,0 +1,395 @@ +import pytest + +from cidr4_merger import ( + Cidr4MergerError, + cidr4_to_node, + find_neighbours_singles, + get_group_with_max_mask_len, + get_net_addr, + get_parent_ip, + have_same_parent, + lift_lonely_node, + make_cidr4, + make_groups, + make_parent, + merge_neighbors, + merge_nodes_cycle, + merge_nodes_deprecated, + merge_nodes_recursion, + reduce_nodes, + sort_nodes, +) + +from .cidr4_data_for_tests import test_cidr4_data + + +def test_true(): + assert True + + +bin_a = "10011000000000001000010000010000" +assert len(bin_a) == 32 +ip_a = int(bin_a, 2) + +bin_b = "10011100000000000000000000101011" +assert len(bin_b) == 32 +ip_b = int(bin_b, 2) + +bin_c = "10011000000000000000000000000000" +assert len(bin_c) == 32 +ip_c = int(bin_c, 2) + +bin_d = "11111100000000000000000000000000" +assert len(bin_c) == 32 +ip_d = int(bin_d, 2) + + +def test_cidr4_to_node(): + assert cidr4_to_node("4.78.139.0/24") == (72256256, 24, 0, 72256000) + assert cidr4_to_node("0.0.0.0/32") == (0, 32, 0, 0) + + assert cidr4_to_node("23.234.30.0/24") == (401219072, 24, 0, 401219072) + assert cidr4_to_node("172.217.0.0/19") == (2899902464, 19, 0, 2899902464) + assert cidr4_to_node("23.225.141.0/24") == (400657664, 24, 0, 400657408) + assert cidr4_to_node("31.13.94.0/23") == (520969728, 23, 0, 520969216) + + assert cidr4_to_node("0.0.0.0/2") == (0, 2, 0, 0) + assert cidr4_to_node("64.0.0.0/2") == (1073741824, 2, 0, 0) + assert cidr4_to_node("128.0.0.0/2") == (2147483648, 2, 0, 2147483648) + assert cidr4_to_node("192.0.0.0/2") == (3221225472, 2, 0, 2147483648) + + +def test_make_cidr4(): + assert make_cidr4(72256256, 24) == "4.78.139.0/24" + assert make_cidr4(0, 32) == "0.0.0.0/32" + + assert make_cidr4(401219072, 24) == "23.234.30.0/24" + assert make_cidr4(2899902464, 19) == "172.217.0.0/19" + assert make_cidr4(400657664, 24) == "23.225.141.0/24" + assert make_cidr4(520969728, 23) == "31.13.94.0/23" + + +def test_get_net_addr(): + assert get_net_addr(ip_a, 5) == ip_c + assert get_net_addr(ip_b, 5) == ip_c + assert get_net_addr(0, 1) == 0 + assert get_net_addr(0, 0) == 0 + + +def test_get_parent_mask(): + assert get_parent_ip(ip_a, 6) == ip_c + assert get_parent_ip(ip_b, 6) == ip_c + assert get_parent_ip(0, 1) == 0 + + with pytest.raises(Exception) as exc_info: + get_parent_ip(0, 0) + assert str(exc_info.value) == "The top of the tree has no parent!" + assert exc_info.type is Cidr4MergerError + + +def test_have_same_parent(): + assert have_same_parent(ip_c, 6, ip_c, 6) is True + assert have_same_parent(ip_c, 6, ip_c, 5) is False + assert have_same_parent(ip_a, 6, ip_d, 6) is False + assert have_same_parent(ip_a, 6, 0, 1) is False + assert have_same_parent(ip_a, 6, 0, 0) is False + + +def test_sort_nodes(): + assert sort_nodes( + [ + (401219072, 24, 0, 401219072), + (2899902464, 19, 0, 2899902464), + (400657664, 24, 0, 400657408), + (520969728, 23, 0, 520969216), + ] + ) == [ + (2899902464, 19, 0, 2899902464), + (520969728, 23, 0, 520969216), + (400657664, 24, 0, 400657408), + (401219072, 24, 0, 401219072), + ] + + +def test_get_group_with_max_mask_len(): + assert get_group_with_max_mask_len( + [ + (2899902464, 19, 0, 2899902464), + (520969728, 23, 0, 520969216), + (400657664, 24, 0, 400657408), + (401219072, 24, 0, 401219072), + ] + ) == [(400657664, 24, 0, 400657408), (401219072, 24, 0, 401219072)] + + assert get_group_with_max_mask_len( + [ + (401219072, 24, 0, 401219072), + (2899902464, 19, 0, 2899902464), + (520969728, 23, 0, 520969216), + ] + ) == [(401219072, 24, 0, 401219072)] + + +def test_make_parent(): + assert make_parent((0, 2, 12, 0), (1073741824, 2, 3, 0)) == (0, 1, 15, 0) + assert make_parent( + (2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648) + ) == (2147483648, 1, 3, 0) + + with pytest.raises(Exception) as exc_info: + make_parent((0, 2, 12, 0), (3221225472, 2, 2, 2147483648)) + assert str(exc_info.value) == "Nodes must be neighbors!" + assert exc_info.type is Cidr4MergerError + + +def test_reduce_nodes(): + assert reduce_nodes( + [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + ] + ) == [ + (0, 1, 15, 0), + ] + + assert reduce_nodes( + [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ] + ) == [ + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + ] + + assert reduce_nodes( + [ + (0, 2, 12, 0), + (2147483648, 1, 0, 0), + ] + ) == [ + (0, 1, 12 + 2**30, 0), + (2147483648, 1, 0, 0), + ] + + with pytest.raises(Exception) as exc_info: + reduce_nodes( + [ + (0, 1, 12 + 2**30, 0), + (2147483648, 1, 0, 0), + ] + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_merge_nodes_deprecated(): + assert merge_nodes_deprecated( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 2, + ) == [ + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + ] + + with pytest.raises(Exception) as exc_info: + merge_nodes_deprecated( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 1, + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_make_groups(): + nodes = [ + (2398793728, 20, 0, 2398789632), + (2899943424, 20, 0, 2899943424), + (3627728896, 20, 0, 3627728896), + (520963072, 22, 0, 520962048), + (1089054720, 22, 0, 1089054720), + (2899902464, 19, 0, 2899902464), + (2915221504, 19, 0, 2915221504), + ] + assert dict(make_groups(nodes)) == { + 20: [ + (2398793728, 20, 0, 2398789632), + (2899943424, 20, 0, 2899943424), + (3627728896, 20, 0, 3627728896), + ], + 22: [(520963072, 22, 0, 520962048), (1089054720, 22, 0, 1089054720)], + 19: [(2899902464, 19, 0, 2899902464), (2915221504, 19, 0, 2915221504)], + } + + +@pytest.fixture +def nodes_only_neighbours(): + return [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ] + + +@pytest.fixture +def nodes_only_singles(): + return [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + ] + + +@pytest.fixture +def nodes_with_neighbours_n_singles(): + return [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + (2147483648, 2, 1, 2147483648), + ] + + +@pytest.fixture +def groups_only_neighbours(nodes_only_neighbours): + return make_groups(nodes_only_neighbours) + + +@pytest.fixture +def groups_only_singles(nodes_only_singles): + return make_groups(nodes_only_singles) + + +@pytest.fixture +def groups_with_neighbours_n_singles(nodes_with_neighbours_n_singles): + return make_groups(nodes_with_neighbours_n_singles) + + +def test_find_neighbours_singles( + groups_only_neighbours, + groups_only_singles, + groups_with_neighbours_n_singles, +): + assert find_neighbours_singles(groups_only_neighbours) == ( + [ + ((0, 2, 12, 0), (1073741824, 2, 3, 0)), + ((2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648)), + ], + [], + ) + + assert find_neighbours_singles(groups_only_singles) == ( + [], + [(0, 2, 12, 0), (2147483648, 2, 1, 2147483648)], + ) + + assert find_neighbours_singles(groups_with_neighbours_n_singles) == ( + [((0, 2, 12, 0), (1073741824, 2, 3, 0))], + [(2147483648, 2, 1, 2147483648)], + ) + + +def test_merge_neighbors__only_neighbours(nodes_only_neighbours): + new_nodes = merge_neighbors( + nodes_only_neighbours, + [ + ((0, 2, 12, 0), (1073741824, 2, 3, 0)), + ((2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648)), + ], + ) + assert new_nodes == [ + (0, 1, 15, 0), + (2147483648, 1, 3, 0), + ] + + +def test_merge_neighbors__neighbours_n_singles(nodes_with_neighbours_n_singles): + new_nodes = merge_neighbors( + nodes_with_neighbours_n_singles, + [((0, 2, 12, 0), (1073741824, 2, 3, 0))], + ) + assert new_nodes == [ + (0, 1, 15, 0), + (2147483648, 2, 1, 2147483648), + ] + + +def test_lift_lonely_node(nodes_only_singles): + singles = [(0, 2, 12, 0), (2147483648, 2, 1, 2147483648)] + new_nodes = lift_lonely_node(nodes_only_singles, singles) + assert new_nodes == [(2147483648, 1, 1073741825, 0), (0, 2, 12, 0)] + + +def test_merge_nodes_recursion(): + assert merge_nodes_recursion( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 2, + ) == [ + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + ] + + with pytest.raises(Exception) as exc_info: + merge_nodes_recursion( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 1, + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_merge_nodes_cycle(): + assert merge_nodes_cycle( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 2, + ) == [ + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + ] + + with pytest.raises(Exception) as exc_info: + merge_nodes_cycle( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 1, + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_merge_nodes_recursion_vs_cycle(): + required_len = 20 + + test_data = test_cidr4_data.strip().splitlines() + test_nodes = list(map(cidr4_to_node, test_data)) + test_nodes = sort_nodes(test_nodes) + + merged_nodes_recursion = merge_nodes_recursion(test_nodes, required_len) + merged_nodes_cycle = merge_nodes_cycle(test_nodes, required_len) + + assert merged_nodes_recursion == merged_nodes_cycle diff --git a/archive/cidr4_merger.py b/archive/cidr4_merger.py new file mode 100644 index 0000000..7dfa3d0 --- /dev/null +++ b/archive/cidr4_merger.py @@ -0,0 +1,226 @@ +import cProfile +import sys +from collections import defaultdict + +sys.setrecursionlimit(10_000) + +Node = tuple[int, int, int, int] + + +class Cidr4MergerError(Exception): + pass + + +def get_data(input_file): + with open(input_file, "r") as file: + data = file.read().splitlines() + return data + + +def cidr4_to_node(cidr4: str) -> Node: + ip_address, mask_len = cidr4.strip().split("/") + mask_len = int(mask_len) + a, b, c, d = list(map(int, ip_address.split("."))) + ip = a * 256**3 + b * 256**2 + c * 256**1 + d * 256**0 + added_ips_number = 0 + parent_ip = get_parent_ip(ip, mask_len) + return ip, mask_len, added_ips_number, parent_ip + + +def sort_nodes(nodes: list[Node]) -> list[Node]: + return sorted(nodes, key=lambda x: (x[1], x[0])) + + +def get_net_addr(ip: int, mask_len: int) -> int: + mask = ((1 << mask_len) - 1) << (32 - mask_len) + net_addr = ip & mask + return net_addr + + +def get_parent_ip(ip: int, mask_len: int) -> int: + if mask_len == 0: + raise Cidr4MergerError("The top of the tree has no parent!") + return get_net_addr(ip, mask_len - 1) + + +def have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b) -> bool: + return mask_len_a == mask_len_b and parent_ip_a == parent_ip_b + + +def get_group_with_max_mask_len(nodes: list[Node]) -> list[Node]: + max_mask_len = max(nodes, key=lambda x: x[1])[1] + return list(filter(lambda x: x[1] == max_mask_len, nodes)) + + +def make_parent(a: Node, b: Node | None = None) -> Node: + ip_a, mask_len_a, added_ips_number_a, parent_ip_a = a + if b: + ip_b, mask_len_b, added_ips_b, parent_ip_b = b + if not have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): + raise Cidr4MergerError("Nodes must be neighbors!") + added_ips_number = added_ips_number_a + added_ips_b + else: + added_ips_number = added_ips_number_a + 2 ** (32 - mask_len_a) + ip = parent_ip_a + mask_len = mask_len_a - 1 + parent_ip = get_parent_ip(ip, mask_len) + return ip, mask_len, added_ips_number, parent_ip + + +def reduce_nodes(nodes: list[Node]) -> list[Node]: + group = get_group_with_max_mask_len(nodes) + + neighbours = [] + loners = [] + i = 0 + while i < len(group) - 1: + a, b = group[i], group[i + 1] + ip_a, mask_len_a, _, parent_ip_a = a + ip_b, mask_len_b, _, parent_ip_b = b + if have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): + neighbours.append((a, b)) + i += 2 + else: + loners.append(a) + i += 1 + if i == len(group) - 1: + loners.append(group[i]) + + if neighbours: + zipped = zip(neighbours, map(lambda x: make_parent(x[0], x[1]), neighbours)) + min_zipped = min(zipped, key=lambda x: x[1][2]) + (a, b), parent = min_zipped + nodes.remove(a) + nodes.remove(b) + nodes.append(parent) + elif loners: + zipped = zip(loners, map(make_parent, loners)) + min_zipped = min(zipped, key=lambda x: x[1][2]) + a, parent = min_zipped + nodes.remove(a) + nodes.append(parent) + else: + assert False, "Error" + + return sort_nodes(nodes) + + +def merge_nodes_deprecated(nodes: list[Node], required_len: int) -> list[Node]: + while len(nodes) > required_len: + nodes = reduce_nodes(nodes) + return nodes + + +def make_cidr4(ip, mask_len) -> str: + lst = [str(ip >> (i << 3) & 0xFF) for i in reversed(range(4))] + ip_address = ".".join(lst) + return f"{ip_address}/{mask_len}" + + +def lift_lonely_node(nodes: list[Node], singles: list[Node]) -> list[Node]: + # find single whose parent has the least added addresses + min_single, min_parent = singles[0], make_parent(singles[0]) + for node in singles[1:]: + parent = make_parent(node) + if parent[2] < min_parent[2]: + min_single, min_parent = node, parent + + nodes.remove(min_single) + nodes.append(min_parent) + nodes = sort_nodes(nodes) + return nodes + + +def merge_neighbors( + nodes: list[Node], neighbours: list[tuple[Node, Node]] +) -> list[Node]: + for a, b in neighbours: + parent = make_parent(a, b) + nodes.remove(a) + nodes.remove(b) + nodes.append(parent) + return sort_nodes(nodes) + + +def find_neighbours_singles(groups: defaultdict) -> tuple[list, list]: + neighbours = [] + singles = [] + for group in groups.values(): + i = 0 + while i < len(group) - 1: + a, b = group[i], group[i + 1] + ip_a, mask_len_a, _, parent_ip_a = a + ip_b, mask_len_b, _, parent_ip_b = b + if have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): + neighbours.append((a, b)) + i += 2 + else: + singles.append(a) + i += 1 + if i == len(group) - 1: + singles.append(group[i]) + return neighbours, singles + + +def make_groups(nodes: list[Node]) -> defaultdict: + groups = defaultdict(list) + for n in nodes: + groups[n[1]].append(n) + return groups + + +def merge_nodes_recursion(nodes: list[Node], required_len: int) -> list[Node]: + if len(nodes) <= required_len: + return nodes + groups = make_groups(nodes) + neighbours, singles = find_neighbours_singles(groups) + if neighbours: + new_nodes = merge_neighbors(nodes, neighbours) + return merge_nodes_recursion(new_nodes, required_len) + new_nodes = lift_lonely_node(nodes, singles) + return merge_nodes_recursion(new_nodes, required_len) + + +def merge_nodes_cycle(nodes_to_merge: list[Node], required_len: int) -> list[Node]: + nodes = [x for x in nodes_to_merge] + while not len(nodes) <= required_len: + groups = make_groups(nodes) + neighbours, singles = find_neighbours_singles(groups) + if neighbours: + nodes = merge_neighbors(nodes, neighbours) + elif singles: + nodes = lift_lonely_node(nodes, singles) + else: + raise Cidr4MergerError("Invalid case!") + return nodes + + +def main(): + file = "cidr4.txt" + required_len = 20 + + data = get_data(file) + nodes = list(map(cidr4_to_node, data)) + + nodes = sort_nodes(nodes) + # merged_nodes = merge_nodes_deprecated(nodes, required_len) + # merged_nodes = merge_nodes_recursion(nodes, required_len) + merged_nodes = merge_nodes_cycle(nodes, required_len) + + cidr4s = [] + sum_added_ips = 0 + for ip_value, mask_len, added_ips, _ in merged_nodes: + cidr4s.append(make_cidr4(ip_value, mask_len)) + sum_added_ips += added_ips + + cidr4s_str = "\n".join(cidr4s) + print( + f"Исходный список длины {len(nodes)} сокращен до {len(cidr4s)}\n" + f"Количество добавленных ip адресов: {sum_added_ips:_}\n" + f"Список объединенных cidr4:\n" + f"{cidr4s_str}" + ) + + +if __name__ == "__main__": + cProfile.run("main()") -- 2.54.0 From dfd6e21f2b92d3c55b9bd088f357ce32108b1486 Mon Sep 17 00:00:00 2001 From: Fedor Lyanguzov Date: Wed, 22 Jan 2025 18:21:39 +0300 Subject: [PATCH 2/3] Fast and precise algorithms, with tests --- .activate | 7 ++++ tests/cidr4_merge/test_solutions.py | 56 +++++++++++++++++++++++++++ vpn_manager/cidr4_merge/fast.py | 43 +++++++++++++++++++++ vpn_manager/cidr4_merge/precise.py | 59 +++++++++++++++++++++++++++++ vpn_manager/cidr4_merge/util.py | 20 ++++++++++ 5 files changed, 185 insertions(+) create mode 100644 .activate create mode 100644 tests/cidr4_merge/test_solutions.py create mode 100644 vpn_manager/cidr4_merge/fast.py create mode 100644 vpn_manager/cidr4_merge/precise.py create mode 100644 vpn_manager/cidr4_merge/util.py diff --git a/.activate b/.activate new file mode 100644 index 0000000..a3607ff --- /dev/null +++ b/.activate @@ -0,0 +1,7 @@ + +alias venv='source .venv/Scripts/activate' +alias check='flake8 vpn_manager' +alias format='black vpn_manager' +alias test='pytest' + +venv diff --git a/tests/cidr4_merge/test_solutions.py b/tests/cidr4_merge/test_solutions.py new file mode 100644 index 0000000..091fb9b --- /dev/null +++ b/tests/cidr4_merge/test_solutions.py @@ -0,0 +1,56 @@ +from vpn_manager.cidr4_merge.fast import solution as fast +from vpn_manager.cidr4_merge.precise import solution as precise, f +from vpn_manager.cidr4_merge.util import * + + +def test_true(): + assert True + +#cidrs = list(map(cidr4_to_node, get_data())) + +def test_fast_single_lifting(): + assert ([(0, 30)], 0) ==\ + fast([(0, 31), (2, 31)], 1) + assert ([(0, 29)], 2) ==\ + fast([(0, 30), (4, 31)], 1) + assert ([(0, 29)], 3) ==\ + fast([(0, 30), (4, 32)], 1) + +def test_fast_double_lifting(): + assert ([(0, 29)], 4) ==\ + fast([(0, 31), (4, 31)], 1) + + +def test_fast_subnets(): + assert ([(0, 30)], 0) ==\ + fast([(0, 30), (0, 31)], 1) + assert ([(0, 29)], 0) ==\ + fast([(0, 29), (4, 31)], 1) + + +def test_precise_single_lifting(): + assert ([(0, 30)], 0) ==\ + precise([(0, 31), (2, 31)], 1) + assert ([(0, 29)], 2) ==\ + precise([(0, 30), (4, 31)], 1) + assert ([(0, 29)], 3) ==\ + precise([(0, 30), (4, 32)], 1) + +def test_precise_double_lifting(): + assert ([(0, 29)], 4) ==\ + precise([(0, 31), (4, 31)], 1) + + +def test_precise_subnets(): + assert ([(0, 30)], 0) ==\ + precise([(0, 30), (0, 31)], 1) + assert ([(0, 29)], 0) ==\ + precise([(0, 29), (4, 31)], 1) + +def test_precise_f_single_lifting(): + assert (0, (0, 30, 0)) ==\ + f((0, 31, 0), (2, 31, 0)) + assert (2, (0, 29, 2)) ==\ + f((0, 30, 0), (4, 31, 0)) + assert (3, (0, 29, 3)) ==\ + f((0, 30, 0), (4, 32, 0)) diff --git a/vpn_manager/cidr4_merge/fast.py b/vpn_manager/cidr4_merge/fast.py new file mode 100644 index 0000000..ca65b46 --- /dev/null +++ b/vpn_manager/cidr4_merge/fast.py @@ -0,0 +1,43 @@ +from heapq import heapify, heappush, heappop +from .util import mask, get_data, cidr4_to_node, make_cidr4 + + +def solution(cidrs, M): + h = [] + d = {} + for ip, l in cidrs: + h.append((32-l, ip, l)) + d[(ip, l)] = 0 + heapify(h) + while len(h)>M: + x1, ip1, l1 = heappop(h) + x2, ip2, l2 = h[0] + if l1==l2 and ip1 & mask[l1-1] == ip2 & mask[l2-1]: + heappop(h) + if (ip1 & mask[l1-1], l1-1) not in d: + heappush(h, (x1+1, ip1 & mask[l1-1], l1-1)) + d[(ip1 & mask[l1-1], l1-1)] = d[(ip1, l1)] + d[(ip2, l2)] + del d[(ip2, l2)] + else: + if (ip1 & mask[l1-1], l1-1) not in d: + heappush(h, (x1+1, ip1 & mask[l1-1], l1-1)) + d[(ip1 & mask[l1-1], l1-1)] = d[(ip1, l1)] + 2**x1 + del d[(ip1, l1)] + s = sum(d.values()) + cidrs = list(d.keys()) + return cidrs, s + + +def main(): + M = 20 + a = get_data() + b = list(map(cidr4_to_node, a)) + cidrs, s = solution(b, M) + cidrs = sorted([make_cidr4(*x) for x in cidrs]) + print(cidrs, s, sep='\n') + + +if __name__=='__main__': + import cProfile + main() + diff --git a/vpn_manager/cidr4_merge/precise.py b/vpn_manager/cidr4_merge/precise.py new file mode 100644 index 0000000..8dcae41 --- /dev/null +++ b/vpn_manager/cidr4_merge/precise.py @@ -0,0 +1,59 @@ +from .util import mask, get_data, cidr4_to_node, make_cidr4 + + +def f(x, y): + t = x + b = y + if x[1]>y[1]: + t = y + b = x + ip1, l1, a1 = t + ip2, l2, a2 = b + if ip1 & mask[l1] == ip2 & mask[l1]: + return (0, t) + t1 = t2 = 0 + while not l1==l2: + t2 += 2**(32-l2) + l2 -= 1 + ip2 = ip2 & mask[l2] + while not ip1 & mask[l1-1] == ip2 & mask[l2-1]: + t1 += 2**(32-l1) + l1 -= 1 + ip1 = ip1 & mask[l1] + t2 += 2**(32-l2) + l2 -= 1 + ip2 = ip2 & mask[l2] + r = (ip1 & mask[l1-1], l1-1, a1+a2+t1+t2) + return (t1+t2, r) + + +def solution(cidrs, M): + cidrs = sorted((ip, l, 0) for ip, l in cidrs) + while len(cidrs)>M: + t = (None, float('+inf'), None) + for i, (x, y) in enumerate(zip(cidrs, cidrs[1:])): + m, r = f(x, y) + if m str: + lst = [str(ip >> (i << 3) & 0xFF) for i in reversed(range(4))] + ip_address = ".".join(lst) + return f"{ip_address}/{mask_len}" -- 2.54.0 From 66346431555871bce3851fd2f94c7ceea615bc9d Mon Sep 17 00:00:00 2001 From: Pavel Patsey Date: Wed, 22 Jan 2025 18:23:02 +0300 Subject: [PATCH 3/3] init new algorithm --- cidr4_merger.py | 162 ++---------------- tests/test_cidr4_merger.py | 326 ++----------------------------------- 2 files changed, 31 insertions(+), 457 deletions(-) diff --git a/cidr4_merger.py b/cidr4_merger.py index 7dfa3d0..fae09ba 100644 --- a/cidr4_merger.py +++ b/cidr4_merger.py @@ -1,10 +1,6 @@ import cProfile -import sys -from collections import defaultdict -sys.setrecursionlimit(10_000) - -Node = tuple[int, int, int, int] +Node = tuple[int, int, int] class Cidr4MergerError(Exception): @@ -23,12 +19,11 @@ def cidr4_to_node(cidr4: str) -> Node: a, b, c, d = list(map(int, ip_address.split("."))) ip = a * 256**3 + b * 256**2 + c * 256**1 + d * 256**0 added_ips_number = 0 - parent_ip = get_parent_ip(ip, mask_len) - return ip, mask_len, added_ips_number, parent_ip + return ip, mask_len, added_ips_number def sort_nodes(nodes: list[Node]) -> list[Node]: - return sorted(nodes, key=lambda x: (x[1], x[0])) + return sorted(nodes) def get_net_addr(ip: int, mask_len: int) -> int: @@ -43,72 +38,9 @@ def get_parent_ip(ip: int, mask_len: int) -> int: return get_net_addr(ip, mask_len - 1) -def have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b) -> bool: - return mask_len_a == mask_len_b and parent_ip_a == parent_ip_b - - -def get_group_with_max_mask_len(nodes: list[Node]) -> list[Node]: - max_mask_len = max(nodes, key=lambda x: x[1])[1] - return list(filter(lambda x: x[1] == max_mask_len, nodes)) - - -def make_parent(a: Node, b: Node | None = None) -> Node: - ip_a, mask_len_a, added_ips_number_a, parent_ip_a = a - if b: - ip_b, mask_len_b, added_ips_b, parent_ip_b = b - if not have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): - raise Cidr4MergerError("Nodes must be neighbors!") - added_ips_number = added_ips_number_a + added_ips_b - else: - added_ips_number = added_ips_number_a + 2 ** (32 - mask_len_a) - ip = parent_ip_a - mask_len = mask_len_a - 1 - parent_ip = get_parent_ip(ip, mask_len) - return ip, mask_len, added_ips_number, parent_ip - - -def reduce_nodes(nodes: list[Node]) -> list[Node]: - group = get_group_with_max_mask_len(nodes) - - neighbours = [] - loners = [] - i = 0 - while i < len(group) - 1: - a, b = group[i], group[i + 1] - ip_a, mask_len_a, _, parent_ip_a = a - ip_b, mask_len_b, _, parent_ip_b = b - if have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): - neighbours.append((a, b)) - i += 2 - else: - loners.append(a) - i += 1 - if i == len(group) - 1: - loners.append(group[i]) - - if neighbours: - zipped = zip(neighbours, map(lambda x: make_parent(x[0], x[1]), neighbours)) - min_zipped = min(zipped, key=lambda x: x[1][2]) - (a, b), parent = min_zipped - nodes.remove(a) - nodes.remove(b) - nodes.append(parent) - elif loners: - zipped = zip(loners, map(make_parent, loners)) - min_zipped = min(zipped, key=lambda x: x[1][2]) - a, parent = min_zipped - nodes.remove(a) - nodes.append(parent) - else: - assert False, "Error" - - return sort_nodes(nodes) - - -def merge_nodes_deprecated(nodes: list[Node], required_len: int) -> list[Node]: - while len(nodes) > required_len: - nodes = reduce_nodes(nodes) - return nodes +def make_parent(a: Node, b: Node) -> Node: + ip, mask_len, added_ips_number = None + return ip, mask_len, added_ips_number def make_cidr4(ip, mask_len) -> str: @@ -117,81 +49,14 @@ def make_cidr4(ip, mask_len) -> str: return f"{ip_address}/{mask_len}" -def lift_lonely_node(nodes: list[Node], singles: list[Node]) -> list[Node]: - # find single whose parent has the least added addresses - min_single, min_parent = singles[0], make_parent(singles[0]) - for node in singles[1:]: - parent = make_parent(node) - if parent[2] < min_parent[2]: - min_single, min_parent = node, parent - - nodes.remove(min_single) - nodes.append(min_parent) - nodes = sort_nodes(nodes) - return nodes - - -def merge_neighbors( - nodes: list[Node], neighbours: list[tuple[Node, Node]] -) -> list[Node]: - for a, b in neighbours: - parent = make_parent(a, b) - nodes.remove(a) - nodes.remove(b) - nodes.append(parent) - return sort_nodes(nodes) - - -def find_neighbours_singles(groups: defaultdict) -> tuple[list, list]: - neighbours = [] - singles = [] - for group in groups.values(): - i = 0 - while i < len(group) - 1: - a, b = group[i], group[i + 1] - ip_a, mask_len_a, _, parent_ip_a = a - ip_b, mask_len_b, _, parent_ip_b = b - if have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): - neighbours.append((a, b)) - i += 2 - else: - singles.append(a) - i += 1 - if i == len(group) - 1: - singles.append(group[i]) - return neighbours, singles - - -def make_groups(nodes: list[Node]) -> defaultdict: - groups = defaultdict(list) - for n in nodes: - groups[n[1]].append(n) - return groups - - -def merge_nodes_recursion(nodes: list[Node], required_len: int) -> list[Node]: - if len(nodes) <= required_len: - return nodes - groups = make_groups(nodes) - neighbours, singles = find_neighbours_singles(groups) - if neighbours: - new_nodes = merge_neighbors(nodes, neighbours) - return merge_nodes_recursion(new_nodes, required_len) - new_nodes = lift_lonely_node(nodes, singles) - return merge_nodes_recursion(new_nodes, required_len) - - -def merge_nodes_cycle(nodes_to_merge: list[Node], required_len: int) -> list[Node]: +def merge_nodes(nodes_to_merge: list[Node], required_len: int) -> list[Node]: nodes = [x for x in nodes_to_merge] - while not len(nodes) <= required_len: - groups = make_groups(nodes) - neighbours, singles = find_neighbours_singles(groups) - if neighbours: - nodes = merge_neighbors(nodes, neighbours) - elif singles: - nodes = lift_lonely_node(nodes, singles) - else: - raise Cidr4MergerError("Invalid case!") + # преобразовать список нод в список туплов: родитель (ip, mask len, added ips), d_ip = кол-во добавляемых адресов + # найти подходящего родителя (с минимальным значением d_ip), затем мержить два узла: + # 1) если в соседних ветках + # 2) если один из узлов находится в подсети у другого + # повторить пока не достигнем нужного кол-ва узлов + return nodes @@ -203,7 +68,6 @@ def main(): nodes = list(map(cidr4_to_node, data)) nodes = sort_nodes(nodes) - # merged_nodes = merge_nodes_deprecated(nodes, required_len) # merged_nodes = merge_nodes_recursion(nodes, required_len) merged_nodes = merge_nodes_cycle(nodes, required_len) diff --git a/tests/test_cidr4_merger.py b/tests/test_cidr4_merger.py index c891934..3b1d4d0 100644 --- a/tests/test_cidr4_merger.py +++ b/tests/test_cidr4_merger.py @@ -3,20 +3,10 @@ import pytest from cidr4_merger import ( Cidr4MergerError, cidr4_to_node, - find_neighbours_singles, - get_group_with_max_mask_len, get_net_addr, get_parent_ip, - have_same_parent, - lift_lonely_node, make_cidr4, - make_groups, make_parent, - merge_neighbors, - merge_nodes_cycle, - merge_nodes_deprecated, - merge_nodes_recursion, - reduce_nodes, sort_nodes, ) @@ -45,18 +35,18 @@ ip_d = int(bin_d, 2) def test_cidr4_to_node(): - assert cidr4_to_node("4.78.139.0/24") == (72256256, 24, 0, 72256000) - assert cidr4_to_node("0.0.0.0/32") == (0, 32, 0, 0) + assert cidr4_to_node("4.78.139.0/24") == (72256256, 24, 0) + assert cidr4_to_node("0.0.0.0/32") == (0, 32, 0) - assert cidr4_to_node("23.234.30.0/24") == (401219072, 24, 0, 401219072) - assert cidr4_to_node("172.217.0.0/19") == (2899902464, 19, 0, 2899902464) - assert cidr4_to_node("23.225.141.0/24") == (400657664, 24, 0, 400657408) - assert cidr4_to_node("31.13.94.0/23") == (520969728, 23, 0, 520969216) + assert cidr4_to_node("23.234.30.0/24") == (401219072, 24, 0) + assert cidr4_to_node("172.217.0.0/19") == (2899902464, 19, 0) + assert cidr4_to_node("23.225.141.0/24") == (400657664, 24, 0) + assert cidr4_to_node("31.13.94.0/23") == (520969728, 23, 0) - assert cidr4_to_node("0.0.0.0/2") == (0, 2, 0, 0) - assert cidr4_to_node("64.0.0.0/2") == (1073741824, 2, 0, 0) - assert cidr4_to_node("128.0.0.0/2") == (2147483648, 2, 0, 2147483648) - assert cidr4_to_node("192.0.0.0/2") == (3221225472, 2, 0, 2147483648) + assert cidr4_to_node("0.0.0.0/2") == (0, 2, 0) + assert cidr4_to_node("64.0.0.0/2") == (1073741824, 2, 0) + assert cidr4_to_node("128.0.0.0/2") == (2147483648, 2, 0) + assert cidr4_to_node("192.0.0.0/2") == (3221225472, 2, 0) def test_make_cidr4(): @@ -87,49 +77,22 @@ def test_get_parent_mask(): assert exc_info.type is Cidr4MergerError -def test_have_same_parent(): - assert have_same_parent(ip_c, 6, ip_c, 6) is True - assert have_same_parent(ip_c, 6, ip_c, 5) is False - assert have_same_parent(ip_a, 6, ip_d, 6) is False - assert have_same_parent(ip_a, 6, 0, 1) is False - assert have_same_parent(ip_a, 6, 0, 0) is False - - def test_sort_nodes(): assert sort_nodes( [ - (401219072, 24, 0, 401219072), - (2899902464, 19, 0, 2899902464), - (400657664, 24, 0, 400657408), - (520969728, 23, 0, 520969216), + (401219072, 24, 0), + (2899902464, 19, 0), + (400657664, 24, 0), + (520969728, 23, 0), ] ) == [ - (2899902464, 19, 0, 2899902464), - (520969728, 23, 0, 520969216), - (400657664, 24, 0, 400657408), - (401219072, 24, 0, 401219072), + (2899902464, 19, 0), + (400657664, 24, 0), + (401219072, 24, 0), + (520969728, 23, 0), ] -def test_get_group_with_max_mask_len(): - assert get_group_with_max_mask_len( - [ - (2899902464, 19, 0, 2899902464), - (520969728, 23, 0, 520969216), - (400657664, 24, 0, 400657408), - (401219072, 24, 0, 401219072), - ] - ) == [(400657664, 24, 0, 400657408), (401219072, 24, 0, 401219072)] - - assert get_group_with_max_mask_len( - [ - (401219072, 24, 0, 401219072), - (2899902464, 19, 0, 2899902464), - (520969728, 23, 0, 520969216), - ] - ) == [(401219072, 24, 0, 401219072)] - - def test_make_parent(): assert make_parent((0, 2, 12, 0), (1073741824, 2, 3, 0)) == (0, 1, 15, 0) assert make_parent( @@ -140,256 +103,3 @@ def test_make_parent(): make_parent((0, 2, 12, 0), (3221225472, 2, 2, 2147483648)) assert str(exc_info.value) == "Nodes must be neighbors!" assert exc_info.type is Cidr4MergerError - - -def test_reduce_nodes(): - assert reduce_nodes( - [ - (0, 2, 12, 0), - (1073741824, 2, 3, 0), - ] - ) == [ - (0, 1, 15, 0), - ] - - assert reduce_nodes( - [ - (0, 2, 12, 0), - (1073741824, 2, 3, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ] - ) == [ - (2147483648, 1, 3, 0), - (0, 2, 12, 0), - (1073741824, 2, 3, 0), - ] - - assert reduce_nodes( - [ - (0, 2, 12, 0), - (2147483648, 1, 0, 0), - ] - ) == [ - (0, 1, 12 + 2**30, 0), - (2147483648, 1, 0, 0), - ] - - with pytest.raises(Exception) as exc_info: - reduce_nodes( - [ - (0, 1, 12 + 2**30, 0), - (2147483648, 1, 0, 0), - ] - ) - assert exc_info.type is Cidr4MergerError - assert str(exc_info.value) == "The top of the tree has no parent!" - - -def test_merge_nodes_deprecated(): - assert merge_nodes_deprecated( - [ - (0, 2, 12, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ], - 2, - ) == [ - (2147483648, 1, 3, 0), - (0, 2, 12, 0), - ] - - with pytest.raises(Exception) as exc_info: - merge_nodes_deprecated( - [ - (0, 2, 12, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ], - 1, - ) - assert exc_info.type is Cidr4MergerError - assert str(exc_info.value) == "The top of the tree has no parent!" - - -def test_make_groups(): - nodes = [ - (2398793728, 20, 0, 2398789632), - (2899943424, 20, 0, 2899943424), - (3627728896, 20, 0, 3627728896), - (520963072, 22, 0, 520962048), - (1089054720, 22, 0, 1089054720), - (2899902464, 19, 0, 2899902464), - (2915221504, 19, 0, 2915221504), - ] - assert dict(make_groups(nodes)) == { - 20: [ - (2398793728, 20, 0, 2398789632), - (2899943424, 20, 0, 2899943424), - (3627728896, 20, 0, 3627728896), - ], - 22: [(520963072, 22, 0, 520962048), (1089054720, 22, 0, 1089054720)], - 19: [(2899902464, 19, 0, 2899902464), (2915221504, 19, 0, 2915221504)], - } - - -@pytest.fixture -def nodes_only_neighbours(): - return [ - (0, 2, 12, 0), - (1073741824, 2, 3, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ] - - -@pytest.fixture -def nodes_only_singles(): - return [ - (0, 2, 12, 0), - (2147483648, 2, 1, 2147483648), - ] - - -@pytest.fixture -def nodes_with_neighbours_n_singles(): - return [ - (0, 2, 12, 0), - (1073741824, 2, 3, 0), - (2147483648, 2, 1, 2147483648), - ] - - -@pytest.fixture -def groups_only_neighbours(nodes_only_neighbours): - return make_groups(nodes_only_neighbours) - - -@pytest.fixture -def groups_only_singles(nodes_only_singles): - return make_groups(nodes_only_singles) - - -@pytest.fixture -def groups_with_neighbours_n_singles(nodes_with_neighbours_n_singles): - return make_groups(nodes_with_neighbours_n_singles) - - -def test_find_neighbours_singles( - groups_only_neighbours, - groups_only_singles, - groups_with_neighbours_n_singles, -): - assert find_neighbours_singles(groups_only_neighbours) == ( - [ - ((0, 2, 12, 0), (1073741824, 2, 3, 0)), - ((2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648)), - ], - [], - ) - - assert find_neighbours_singles(groups_only_singles) == ( - [], - [(0, 2, 12, 0), (2147483648, 2, 1, 2147483648)], - ) - - assert find_neighbours_singles(groups_with_neighbours_n_singles) == ( - [((0, 2, 12, 0), (1073741824, 2, 3, 0))], - [(2147483648, 2, 1, 2147483648)], - ) - - -def test_merge_neighbors__only_neighbours(nodes_only_neighbours): - new_nodes = merge_neighbors( - nodes_only_neighbours, - [ - ((0, 2, 12, 0), (1073741824, 2, 3, 0)), - ((2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648)), - ], - ) - assert new_nodes == [ - (0, 1, 15, 0), - (2147483648, 1, 3, 0), - ] - - -def test_merge_neighbors__neighbours_n_singles(nodes_with_neighbours_n_singles): - new_nodes = merge_neighbors( - nodes_with_neighbours_n_singles, - [((0, 2, 12, 0), (1073741824, 2, 3, 0))], - ) - assert new_nodes == [ - (0, 1, 15, 0), - (2147483648, 2, 1, 2147483648), - ] - - -def test_lift_lonely_node(nodes_only_singles): - singles = [(0, 2, 12, 0), (2147483648, 2, 1, 2147483648)] - new_nodes = lift_lonely_node(nodes_only_singles, singles) - assert new_nodes == [(2147483648, 1, 1073741825, 0), (0, 2, 12, 0)] - - -def test_merge_nodes_recursion(): - assert merge_nodes_recursion( - [ - (0, 2, 12, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ], - 2, - ) == [ - (2147483648, 1, 3, 0), - (0, 2, 12, 0), - ] - - with pytest.raises(Exception) as exc_info: - merge_nodes_recursion( - [ - (0, 2, 12, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ], - 1, - ) - assert exc_info.type is Cidr4MergerError - assert str(exc_info.value) == "The top of the tree has no parent!" - - -def test_merge_nodes_cycle(): - assert merge_nodes_cycle( - [ - (0, 2, 12, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ], - 2, - ) == [ - (2147483648, 1, 3, 0), - (0, 2, 12, 0), - ] - - with pytest.raises(Exception) as exc_info: - merge_nodes_cycle( - [ - (0, 2, 12, 0), - (2147483648, 2, 1, 2147483648), - (3221225472, 2, 2, 2147483648), - ], - 1, - ) - assert exc_info.type is Cidr4MergerError - assert str(exc_info.value) == "The top of the tree has no parent!" - - -def test_merge_nodes_recursion_vs_cycle(): - required_len = 20 - - test_data = test_cidr4_data.strip().splitlines() - test_nodes = list(map(cidr4_to_node, test_data)) - test_nodes = sort_nodes(test_nodes) - - merged_nodes_recursion = merge_nodes_recursion(test_nodes, required_len) - merged_nodes_cycle = merge_nodes_cycle(test_nodes, required_len) - - assert merged_nodes_recursion == merged_nodes_cycle -- 2.54.0