diff --git a/archive/_test_cidr4_merger.py b/archive/_test_cidr4_merger.py new file mode 100644 index 0000000..c891934 --- /dev/null +++ b/archive/_test_cidr4_merger.py @@ -0,0 +1,395 @@ +import pytest + +from cidr4_merger import ( + Cidr4MergerError, + cidr4_to_node, + find_neighbours_singles, + get_group_with_max_mask_len, + get_net_addr, + get_parent_ip, + have_same_parent, + lift_lonely_node, + make_cidr4, + make_groups, + make_parent, + merge_neighbors, + merge_nodes_cycle, + merge_nodes_deprecated, + merge_nodes_recursion, + reduce_nodes, + sort_nodes, +) + +from .cidr4_data_for_tests import test_cidr4_data + + +def test_true(): + assert True + + +bin_a = "10011000000000001000010000010000" +assert len(bin_a) == 32 +ip_a = int(bin_a, 2) + +bin_b = "10011100000000000000000000101011" +assert len(bin_b) == 32 +ip_b = int(bin_b, 2) + +bin_c = "10011000000000000000000000000000" +assert len(bin_c) == 32 +ip_c = int(bin_c, 2) + +bin_d = "11111100000000000000000000000000" +assert len(bin_c) == 32 +ip_d = int(bin_d, 2) + + +def test_cidr4_to_node(): + assert cidr4_to_node("4.78.139.0/24") == (72256256, 24, 0, 72256000) + assert cidr4_to_node("0.0.0.0/32") == (0, 32, 0, 0) + + assert cidr4_to_node("23.234.30.0/24") == (401219072, 24, 0, 401219072) + assert cidr4_to_node("172.217.0.0/19") == (2899902464, 19, 0, 2899902464) + assert cidr4_to_node("23.225.141.0/24") == (400657664, 24, 0, 400657408) + assert cidr4_to_node("31.13.94.0/23") == (520969728, 23, 0, 520969216) + + assert cidr4_to_node("0.0.0.0/2") == (0, 2, 0, 0) + assert cidr4_to_node("64.0.0.0/2") == (1073741824, 2, 0, 0) + assert cidr4_to_node("128.0.0.0/2") == (2147483648, 2, 0, 2147483648) + assert cidr4_to_node("192.0.0.0/2") == (3221225472, 2, 0, 2147483648) + + +def test_make_cidr4(): + assert make_cidr4(72256256, 24) == "4.78.139.0/24" + assert make_cidr4(0, 32) == "0.0.0.0/32" + + assert make_cidr4(401219072, 24) == "23.234.30.0/24" + assert make_cidr4(2899902464, 19) == "172.217.0.0/19" + assert make_cidr4(400657664, 24) == "23.225.141.0/24" + assert make_cidr4(520969728, 23) == "31.13.94.0/23" + + +def test_get_net_addr(): + assert get_net_addr(ip_a, 5) == ip_c + assert get_net_addr(ip_b, 5) == ip_c + assert get_net_addr(0, 1) == 0 + assert get_net_addr(0, 0) == 0 + + +def test_get_parent_mask(): + assert get_parent_ip(ip_a, 6) == ip_c + assert get_parent_ip(ip_b, 6) == ip_c + assert get_parent_ip(0, 1) == 0 + + with pytest.raises(Exception) as exc_info: + get_parent_ip(0, 0) + assert str(exc_info.value) == "The top of the tree has no parent!" + assert exc_info.type is Cidr4MergerError + + +def test_have_same_parent(): + assert have_same_parent(ip_c, 6, ip_c, 6) is True + assert have_same_parent(ip_c, 6, ip_c, 5) is False + assert have_same_parent(ip_a, 6, ip_d, 6) is False + assert have_same_parent(ip_a, 6, 0, 1) is False + assert have_same_parent(ip_a, 6, 0, 0) is False + + +def test_sort_nodes(): + assert sort_nodes( + [ + (401219072, 24, 0, 401219072), + (2899902464, 19, 0, 2899902464), + (400657664, 24, 0, 400657408), + (520969728, 23, 0, 520969216), + ] + ) == [ + (2899902464, 19, 0, 2899902464), + (520969728, 23, 0, 520969216), + (400657664, 24, 0, 400657408), + (401219072, 24, 0, 401219072), + ] + + +def test_get_group_with_max_mask_len(): + assert get_group_with_max_mask_len( + [ + (2899902464, 19, 0, 2899902464), + (520969728, 23, 0, 520969216), + (400657664, 24, 0, 400657408), + (401219072, 24, 0, 401219072), + ] + ) == [(400657664, 24, 0, 400657408), (401219072, 24, 0, 401219072)] + + assert get_group_with_max_mask_len( + [ + (401219072, 24, 0, 401219072), + (2899902464, 19, 0, 2899902464), + (520969728, 23, 0, 520969216), + ] + ) == [(401219072, 24, 0, 401219072)] + + +def test_make_parent(): + assert make_parent((0, 2, 12, 0), (1073741824, 2, 3, 0)) == (0, 1, 15, 0) + assert make_parent( + (2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648) + ) == (2147483648, 1, 3, 0) + + with pytest.raises(Exception) as exc_info: + make_parent((0, 2, 12, 0), (3221225472, 2, 2, 2147483648)) + assert str(exc_info.value) == "Nodes must be neighbors!" + assert exc_info.type is Cidr4MergerError + + +def test_reduce_nodes(): + assert reduce_nodes( + [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + ] + ) == [ + (0, 1, 15, 0), + ] + + assert reduce_nodes( + [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ] + ) == [ + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + ] + + assert reduce_nodes( + [ + (0, 2, 12, 0), + (2147483648, 1, 0, 0), + ] + ) == [ + (0, 1, 12 + 2**30, 0), + (2147483648, 1, 0, 0), + ] + + with pytest.raises(Exception) as exc_info: + reduce_nodes( + [ + (0, 1, 12 + 2**30, 0), + (2147483648, 1, 0, 0), + ] + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_merge_nodes_deprecated(): + assert merge_nodes_deprecated( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 2, + ) == [ + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + ] + + with pytest.raises(Exception) as exc_info: + merge_nodes_deprecated( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 1, + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_make_groups(): + nodes = [ + (2398793728, 20, 0, 2398789632), + (2899943424, 20, 0, 2899943424), + (3627728896, 20, 0, 3627728896), + (520963072, 22, 0, 520962048), + (1089054720, 22, 0, 1089054720), + (2899902464, 19, 0, 2899902464), + (2915221504, 19, 0, 2915221504), + ] + assert dict(make_groups(nodes)) == { + 20: [ + (2398793728, 20, 0, 2398789632), + (2899943424, 20, 0, 2899943424), + (3627728896, 20, 0, 3627728896), + ], + 22: [(520963072, 22, 0, 520962048), (1089054720, 22, 0, 1089054720)], + 19: [(2899902464, 19, 0, 2899902464), (2915221504, 19, 0, 2915221504)], + } + + +@pytest.fixture +def nodes_only_neighbours(): + return [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ] + + +@pytest.fixture +def nodes_only_singles(): + return [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + ] + + +@pytest.fixture +def nodes_with_neighbours_n_singles(): + return [ + (0, 2, 12, 0), + (1073741824, 2, 3, 0), + (2147483648, 2, 1, 2147483648), + ] + + +@pytest.fixture +def groups_only_neighbours(nodes_only_neighbours): + return make_groups(nodes_only_neighbours) + + +@pytest.fixture +def groups_only_singles(nodes_only_singles): + return make_groups(nodes_only_singles) + + +@pytest.fixture +def groups_with_neighbours_n_singles(nodes_with_neighbours_n_singles): + return make_groups(nodes_with_neighbours_n_singles) + + +def test_find_neighbours_singles( + groups_only_neighbours, + groups_only_singles, + groups_with_neighbours_n_singles, +): + assert find_neighbours_singles(groups_only_neighbours) == ( + [ + ((0, 2, 12, 0), (1073741824, 2, 3, 0)), + ((2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648)), + ], + [], + ) + + assert find_neighbours_singles(groups_only_singles) == ( + [], + [(0, 2, 12, 0), (2147483648, 2, 1, 2147483648)], + ) + + assert find_neighbours_singles(groups_with_neighbours_n_singles) == ( + [((0, 2, 12, 0), (1073741824, 2, 3, 0))], + [(2147483648, 2, 1, 2147483648)], + ) + + +def test_merge_neighbors__only_neighbours(nodes_only_neighbours): + new_nodes = merge_neighbors( + nodes_only_neighbours, + [ + ((0, 2, 12, 0), (1073741824, 2, 3, 0)), + ((2147483648, 2, 1, 2147483648), (3221225472, 2, 2, 2147483648)), + ], + ) + assert new_nodes == [ + (0, 1, 15, 0), + (2147483648, 1, 3, 0), + ] + + +def test_merge_neighbors__neighbours_n_singles(nodes_with_neighbours_n_singles): + new_nodes = merge_neighbors( + nodes_with_neighbours_n_singles, + [((0, 2, 12, 0), (1073741824, 2, 3, 0))], + ) + assert new_nodes == [ + (0, 1, 15, 0), + (2147483648, 2, 1, 2147483648), + ] + + +def test_lift_lonely_node(nodes_only_singles): + singles = [(0, 2, 12, 0), (2147483648, 2, 1, 2147483648)] + new_nodes = lift_lonely_node(nodes_only_singles, singles) + assert new_nodes == [(2147483648, 1, 1073741825, 0), (0, 2, 12, 0)] + + +def test_merge_nodes_recursion(): + assert merge_nodes_recursion( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 2, + ) == [ + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + ] + + with pytest.raises(Exception) as exc_info: + merge_nodes_recursion( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 1, + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_merge_nodes_cycle(): + assert merge_nodes_cycle( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 2, + ) == [ + (2147483648, 1, 3, 0), + (0, 2, 12, 0), + ] + + with pytest.raises(Exception) as exc_info: + merge_nodes_cycle( + [ + (0, 2, 12, 0), + (2147483648, 2, 1, 2147483648), + (3221225472, 2, 2, 2147483648), + ], + 1, + ) + assert exc_info.type is Cidr4MergerError + assert str(exc_info.value) == "The top of the tree has no parent!" + + +def test_merge_nodes_recursion_vs_cycle(): + required_len = 20 + + test_data = test_cidr4_data.strip().splitlines() + test_nodes = list(map(cidr4_to_node, test_data)) + test_nodes = sort_nodes(test_nodes) + + merged_nodes_recursion = merge_nodes_recursion(test_nodes, required_len) + merged_nodes_cycle = merge_nodes_cycle(test_nodes, required_len) + + assert merged_nodes_recursion == merged_nodes_cycle diff --git a/archive/cidr4_merger.py b/archive/cidr4_merger.py new file mode 100644 index 0000000..7dfa3d0 --- /dev/null +++ b/archive/cidr4_merger.py @@ -0,0 +1,226 @@ +import cProfile +import sys +from collections import defaultdict + +sys.setrecursionlimit(10_000) + +Node = tuple[int, int, int, int] + + +class Cidr4MergerError(Exception): + pass + + +def get_data(input_file): + with open(input_file, "r") as file: + data = file.read().splitlines() + return data + + +def cidr4_to_node(cidr4: str) -> Node: + ip_address, mask_len = cidr4.strip().split("/") + mask_len = int(mask_len) + a, b, c, d = list(map(int, ip_address.split("."))) + ip = a * 256**3 + b * 256**2 + c * 256**1 + d * 256**0 + added_ips_number = 0 + parent_ip = get_parent_ip(ip, mask_len) + return ip, mask_len, added_ips_number, parent_ip + + +def sort_nodes(nodes: list[Node]) -> list[Node]: + return sorted(nodes, key=lambda x: (x[1], x[0])) + + +def get_net_addr(ip: int, mask_len: int) -> int: + mask = ((1 << mask_len) - 1) << (32 - mask_len) + net_addr = ip & mask + return net_addr + + +def get_parent_ip(ip: int, mask_len: int) -> int: + if mask_len == 0: + raise Cidr4MergerError("The top of the tree has no parent!") + return get_net_addr(ip, mask_len - 1) + + +def have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b) -> bool: + return mask_len_a == mask_len_b and parent_ip_a == parent_ip_b + + +def get_group_with_max_mask_len(nodes: list[Node]) -> list[Node]: + max_mask_len = max(nodes, key=lambda x: x[1])[1] + return list(filter(lambda x: x[1] == max_mask_len, nodes)) + + +def make_parent(a: Node, b: Node | None = None) -> Node: + ip_a, mask_len_a, added_ips_number_a, parent_ip_a = a + if b: + ip_b, mask_len_b, added_ips_b, parent_ip_b = b + if not have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): + raise Cidr4MergerError("Nodes must be neighbors!") + added_ips_number = added_ips_number_a + added_ips_b + else: + added_ips_number = added_ips_number_a + 2 ** (32 - mask_len_a) + ip = parent_ip_a + mask_len = mask_len_a - 1 + parent_ip = get_parent_ip(ip, mask_len) + return ip, mask_len, added_ips_number, parent_ip + + +def reduce_nodes(nodes: list[Node]) -> list[Node]: + group = get_group_with_max_mask_len(nodes) + + neighbours = [] + loners = [] + i = 0 + while i < len(group) - 1: + a, b = group[i], group[i + 1] + ip_a, mask_len_a, _, parent_ip_a = a + ip_b, mask_len_b, _, parent_ip_b = b + if have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): + neighbours.append((a, b)) + i += 2 + else: + loners.append(a) + i += 1 + if i == len(group) - 1: + loners.append(group[i]) + + if neighbours: + zipped = zip(neighbours, map(lambda x: make_parent(x[0], x[1]), neighbours)) + min_zipped = min(zipped, key=lambda x: x[1][2]) + (a, b), parent = min_zipped + nodes.remove(a) + nodes.remove(b) + nodes.append(parent) + elif loners: + zipped = zip(loners, map(make_parent, loners)) + min_zipped = min(zipped, key=lambda x: x[1][2]) + a, parent = min_zipped + nodes.remove(a) + nodes.append(parent) + else: + assert False, "Error" + + return sort_nodes(nodes) + + +def merge_nodes_deprecated(nodes: list[Node], required_len: int) -> list[Node]: + while len(nodes) > required_len: + nodes = reduce_nodes(nodes) + return nodes + + +def make_cidr4(ip, mask_len) -> str: + lst = [str(ip >> (i << 3) & 0xFF) for i in reversed(range(4))] + ip_address = ".".join(lst) + return f"{ip_address}/{mask_len}" + + +def lift_lonely_node(nodes: list[Node], singles: list[Node]) -> list[Node]: + # find single whose parent has the least added addresses + min_single, min_parent = singles[0], make_parent(singles[0]) + for node in singles[1:]: + parent = make_parent(node) + if parent[2] < min_parent[2]: + min_single, min_parent = node, parent + + nodes.remove(min_single) + nodes.append(min_parent) + nodes = sort_nodes(nodes) + return nodes + + +def merge_neighbors( + nodes: list[Node], neighbours: list[tuple[Node, Node]] +) -> list[Node]: + for a, b in neighbours: + parent = make_parent(a, b) + nodes.remove(a) + nodes.remove(b) + nodes.append(parent) + return sort_nodes(nodes) + + +def find_neighbours_singles(groups: defaultdict) -> tuple[list, list]: + neighbours = [] + singles = [] + for group in groups.values(): + i = 0 + while i < len(group) - 1: + a, b = group[i], group[i + 1] + ip_a, mask_len_a, _, parent_ip_a = a + ip_b, mask_len_b, _, parent_ip_b = b + if have_same_parent(mask_len_a, parent_ip_a, mask_len_b, parent_ip_b): + neighbours.append((a, b)) + i += 2 + else: + singles.append(a) + i += 1 + if i == len(group) - 1: + singles.append(group[i]) + return neighbours, singles + + +def make_groups(nodes: list[Node]) -> defaultdict: + groups = defaultdict(list) + for n in nodes: + groups[n[1]].append(n) + return groups + + +def merge_nodes_recursion(nodes: list[Node], required_len: int) -> list[Node]: + if len(nodes) <= required_len: + return nodes + groups = make_groups(nodes) + neighbours, singles = find_neighbours_singles(groups) + if neighbours: + new_nodes = merge_neighbors(nodes, neighbours) + return merge_nodes_recursion(new_nodes, required_len) + new_nodes = lift_lonely_node(nodes, singles) + return merge_nodes_recursion(new_nodes, required_len) + + +def merge_nodes_cycle(nodes_to_merge: list[Node], required_len: int) -> list[Node]: + nodes = [x for x in nodes_to_merge] + while not len(nodes) <= required_len: + groups = make_groups(nodes) + neighbours, singles = find_neighbours_singles(groups) + if neighbours: + nodes = merge_neighbors(nodes, neighbours) + elif singles: + nodes = lift_lonely_node(nodes, singles) + else: + raise Cidr4MergerError("Invalid case!") + return nodes + + +def main(): + file = "cidr4.txt" + required_len = 20 + + data = get_data(file) + nodes = list(map(cidr4_to_node, data)) + + nodes = sort_nodes(nodes) + # merged_nodes = merge_nodes_deprecated(nodes, required_len) + # merged_nodes = merge_nodes_recursion(nodes, required_len) + merged_nodes = merge_nodes_cycle(nodes, required_len) + + cidr4s = [] + sum_added_ips = 0 + for ip_value, mask_len, added_ips, _ in merged_nodes: + cidr4s.append(make_cidr4(ip_value, mask_len)) + sum_added_ips += added_ips + + cidr4s_str = "\n".join(cidr4s) + print( + f"Исходный список длины {len(nodes)} сокращен до {len(cidr4s)}\n" + f"Количество добавленных ip адресов: {sum_added_ips:_}\n" + f"Список объединенных cidr4:\n" + f"{cidr4s_str}" + ) + + +if __name__ == "__main__": + cProfile.run("main()")