Mercurial > public > bitcaviar-plus
changeset 28:30535f42d0ff
refactor code
author | Dennis Concepcion Martin <dennisconcepcionmartin@gmail.com> |
---|---|
date | Wed, 02 Feb 2022 21:16:10 +0100 |
parents | 348a07008703 |
children | d9537541d623 |
files | .github/workflows/python-publish.yml Dockerfile README.md src/bitcaviar_plus/block.py src/bitcaviar_plus/helpers.py src/bitcaviar_plus/search.py tests/implementation_testing.py tests/test_block.py tests/test_implementation.py tests/test_unit.py |
diffstat | 10 files changed, 239 insertions(+), 131 deletions(-) [+] |
line wrap: on
line diff
--- a/.github/workflows/python-publish.yml Thu Jan 06 12:10:25 2022 +0000 +++ b/.github/workflows/python-publish.yml Wed Feb 02 21:16:10 2022 +0100 @@ -27,8 +27,8 @@ flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test - run: python -m unittest discover + - name: Testing units + run: python -m unittest tests.test_unit - name: Build package run: python -m build - name: Publish package
--- a/Dockerfile Thu Jan 06 12:10:25 2022 +0000 +++ b/Dockerfile Wed Feb 02 21:16:10 2022 +0100 @@ -13,4 +13,4 @@ RUN ["python", "setup.py", "install"] # Test package -CMD ["python3", "-u", "tests/implementation_testing.py"] \ No newline at end of file +CMD ["python", "-m", "unittest", "tests.test_implementation"] \ No newline at end of file
--- a/README.md Thu Jan 06 12:10:25 2022 +0000 +++ b/README.md Wed Feb 02 21:16:10 2022 +0100 @@ -5,16 +5,27 @@ # bitcaviar-plus + A Bitcoin blockchain parser written in Python. ## Installation -Clone repository + +### Recommended + ```bash pip install bitcaviar-plus ``` +### Manual + +```bash +python setup.py install +``` + ## Usage -### Deserialize first block from file `blk00000.dat` + +### Deserialize Genesis block + ```python from bitcaviar_plus.block import deserialize_block @@ -26,6 +37,7 @@ ``` ### Deserialize entire blockchain + ```python import os from bitcaviar_plus.block import deserialize_block @@ -47,6 +59,7 @@ ``` ### Example output + ```json { "magic_number":"f9beb4d9",
--- a/src/bitcaviar_plus/block.py Thu Jan 06 12:10:25 2022 +0000 +++ b/src/bitcaviar_plus/block.py Wed Feb 02 21:16:10 2022 +0100 @@ -1,11 +1,10 @@ - """ Deserialize methods for blocks """ from bitcaviar_plus.block_structure import * -from bitcaviar_plus.helpers import __get_var_int -from bitcaviar_plus.helpers import __compute_hash +from bitcaviar_plus.helpers import get_var_int +from bitcaviar_plus.helpers import compute_hash from bitcaviar_plus.errors import InvalidMagicBytes @@ -23,12 +22,12 @@ raise InvalidMagicBytes(block.magic_number) block.size = f.read(4)[::-1].hex() - block_header, block.id = __deserialize_header(f) - block.transaction_count = __get_var_int(f) + block_header, block.id = deserialize_header(f) + block.transaction_count = get_var_int(f) transactions = [] for transaction_number in range(int(block.transaction_count, 16)): - transactions.append(__deserialize_transaction_data(f)) + transactions.append(deserialize_transaction_data(f)) block_dict = block.__dict__ block_dict['header'] = block_header @@ -37,7 +36,9 @@ return block_dict -def __deserialize_header(f): +# ---- SECONDARY METHODS ---- + +def deserialize_header(f): """ Deserialize block header More info: https://learnmeabitcoin.com/technical/block-header @@ -48,7 +49,7 @@ # Compute block hash before = f.tell() header = f.read(80) - block_hash = __compute_hash(header) + block_hash = compute_hash(header) f.seek(before) header = Header() @@ -62,7 +63,7 @@ return header.__dict__, block_hash -def __deserialize_transaction_data(f): +def deserialize_transaction_data(f): """ Deserialize transaction data More info: https://learnmeabitcoin.com/technical/transaction-data @@ -73,25 +74,25 @@ transaction = Transaction() start_transaction_data = f.tell() transaction.version = f.read(4)[::-1].hex() - transaction.input_count = __get_var_int(f) + transaction.input_count = get_var_int(f) transaction_inputs = [] for input_number in range(int(transaction.input_count, 16)): transaction_input = TransactionInput() transaction_input.id = f.read(32)[::-1].hex() transaction_input.vout = f.read(4)[::-1].hex() - transaction_input.script_sig_size = __get_var_int(f) + transaction_input.script_sig_size = get_var_int(f) transaction_input.script_sig = f.read(int(transaction_input.script_sig_size, 16)).hex() transaction_input.sequence = f.read(4)[::-1].hex() transaction_inputs.append(transaction_input.__dict__) - transaction.output_count = __get_var_int(f) + transaction.output_count = get_var_int(f) transaction_outputs = [] for output_number in range(int(transaction.output_count, 16)): transaction_output = TransactionOutput() transaction_output.value = f.read(8)[::-1].hex() - transaction_output.script_pub_key_size = __get_var_int(f) + transaction_output.script_pub_key_size = get_var_int(f) transaction_output.script_pub_key = f.read(int(transaction_output.script_pub_key_size, 16)).hex() transaction_outputs.append(transaction_output.__dict__) @@ -103,7 +104,7 @@ f.seek(start_transaction_data) transaction_data = f.read(transaction_data_size) f.seek(end_transaction_data) - transaction.id = __compute_hash(transaction_data) + transaction.id = compute_hash(transaction_data) transaction_dict = transaction.__dict__ transaction_dict['inputs'] = transaction_inputs
--- a/src/bitcaviar_plus/helpers.py Thu Jan 06 12:10:25 2022 +0000 +++ b/src/bitcaviar_plus/helpers.py Wed Feb 02 21:16:10 2022 +0100 @@ -5,7 +5,7 @@ """ -def __get_var_int(f): +def get_var_int(f): """ A VarInt (variable integer) is a field used in transaction data to indicate the number of upcoming fields, or the length of an upcoming field. @@ -28,7 +28,7 @@ return number_of_transactions -def __compute_hash(data): +def compute_hash(data): """ Get hash :param data: bytes, required
--- a/src/bitcaviar_plus/search.py Thu Jan 06 12:10:25 2022 +0000 +++ b/src/bitcaviar_plus/search.py Wed Feb 02 21:16:10 2022 +0100 @@ -1,4 +1,74 @@ - """ Search methods for LEVELDB database -""" \ No newline at end of file +""" + +try: + # noinspection PyUnresolvedReferences + import plyvel +except ImportError: + # Avoid import error running unit tests + print("Couldn't import plyvel package. Are you running unit tests?") + +import tempfile + + +def search_block_with(block_hash): + """ + Search block with a given hash and get value + :param block_hash: string, required + :return: string + """ + + db = level_db_connect() + search_type = bytes.fromhex('62') # 'b' (block) in hex is 62 + block_hash = bytes.fromhex(block_hash)[::-1] + key = search_type + block_hash + value = db.get(key) + db.close() + + return value.hex() + + +# ---- SECONDARY METHODS ---- + +def deserialize_block_search(f): + """ + Deserialize value (block search) + More info: https://bitcoin.stackexchange.com/questions/67515/format-of-a-block-keys-contents-in-bitcoinds-leveldb + :param f: buffer, required + :return: dict + """ + + client_number = f.read(3) + print('Client number: {}'.format(client_number.hex())) + block_height = f.read(1) # Var int 128? + print('Block height: {}'.format(block_height.hex())) + status = f.read(1) # var int 128? + print('Status: {}'.format(status.hex())) + number_of_transactions = f.read(1) # var int 128? + print('Number of transactions: {}'.format(number_of_transactions.hex())) + + +def create_file_with(binary): + with tempfile.TemporaryFile() as fp: + fp.write(binary) + fp.seek(0) + + return fp + + +# noinspection PyUnresolvedReferences +def level_db_connect(): + db = plyvel.DB('/bitcoin-node/.bitcoin/blocks/index/', create_if_missing=False) + + return db + + +def get_128_var_int(f): + """ + This var int is different from helpers.get_var_int + :param f: + :return: string + """ + + pass
--- a/tests/implementation_testing.py Thu Jan 06 12:10:25 2022 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,46 +0,0 @@ -import os -import plyvel -import tempfile -from bitcaviar_plus.block import deserialize_block -from bitcaviar_plus.block import __deserialize_header -from bitcaviar_plus.errors import InvalidMagicBytes - - -def parse_genesis_block(): - blk_path = '/bitcoin-node/.bitcoin/blocks/blk00355.dat' - - with open(blk_path, 'rb') as f: - file_size = os.path.getsize(blk_path) - while f.tell() < file_size: - try: - block = deserialize_block(f) - except InvalidMagicBytes as e: - print(e) - - -# noinspection PyUnresolvedReferences -def iterate_leveldb_keys(): - db = plyvel.DB('/bitcoin-node/.bitcoin/blocks/index/', create_if_missing=False) - for key, value in db: - print('---- RAW KEY ----') - print(key.hex()) - print('---- LITTLE ENDIAN KEY ----') - print(key[::-1].hex()) - print('---- RAW VALUE ----') - print(value[::-1].hex()) - exit() - - -# noinspection PyUnresolvedReferences -def search_block(): - db = plyvel.DB('/bitcoin-node/.bitcoin/blocks/index/', create_if_missing=False) - search_type = bytes.fromhex('62') # 'b' (block) in hex is 62 - block_hash = bytes.fromhex('000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f')[::-1] - key = search_type + block_hash - value = db.get(key) - print(value.hex()) - db.close() - - -if __name__ == '__main__': - search_block()
--- a/tests/test_block.py Thu Jan 06 12:10:25 2022 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,62 +0,0 @@ -from unittest import TestCase -from testfixtures import TempDirectory -from bitcaviar_plus.block import deserialize_block - - -class TestBlock(TestCase): - genesis_block_hex = """ - f9beb4d91d0100000100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67 - 768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c0101000000010000000000000000000000000000000000000000 - 000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72 - 206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0 - fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6b - f11d5fac00000000 - """ - - expected_genesis_block = { - 'magic_number': 'f9beb4d9', - 'size': '0000011d', - 'id': '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', - 'transaction_count': '01', - 'header': { - 'version': '00000001', - 'previous_block_id': '0000000000000000000000000000000000000000000000000000000000000000', - 'merkle_root': '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b', - 'time': '495fab29', - 'bits': '1d00ffff', - 'nonce': '7c2bac1d' - }, - 'transactions': [{ - 'version': '00000001', - 'input_count': '01', - 'output_count': '01', - 'lock_time': '00000000', - 'id': '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b', - 'inputs': [{ - 'id': '0000000000000000000000000000000000000000000000000000000000000000', - 'vout': 'ffffffff', - 'script_sig_size': '4d', - 'script_sig': '04ffff001d0104455468652054696d65732030332f4a616e' - '2f32303039204368616e63656c6c6f72206f6e206272696e' - '6b206f66207365636f6e64206261696c6f757420666f722062616e6b73', - 'sequence': 'ffffffff' - }], - 'outputs': [{ - 'value': '000000012a05f200', - 'script_pub_key_size': '43', - 'script_pub_key': '4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f' - '6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac' - }] - }] - } - - def test_deserialize_block(self): - with TempDirectory() as d: - test_block_binary = bytes.fromhex(self.genesis_block_hex) - d.write('test_block.dat', test_block_binary) - - with open('{}/test_block.dat'.format(d.path), 'rb') as f: - block = deserialize_block(f) - self.assertEqual( - block, self.expected_genesis_block, 'Genesis block is not equal to expected genesis block' - )
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/test_implementation.py Wed Feb 02 21:16:10 2022 +0100 @@ -0,0 +1,68 @@ +from unittest import TestCase +from bitcaviar_plus.block import deserialize_block +from bitcaviar_plus.errors import InvalidMagicBytes +from bitcaviar_plus.search import search_block_with + + +class TestBlockImplementation(TestCase): + expected_genesis_block = { + 'magic_number': 'f9beb4d9', + 'size': '0000011d', + 'id': '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f', + 'transaction_count': '01', + 'header': { + 'version': '00000001', + 'previous_block_id': '0000000000000000000000000000000000000000000000000000000000000000', + 'merkle_root': '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b', + 'time': '495fab29', + 'bits': '1d00ffff', + 'nonce': '7c2bac1d' + }, + 'transactions': [{ + 'version': '00000001', + 'input_count': '01', + 'output_count': '01', + 'lock_time': '00000000', + 'id': '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b', + 'inputs': [{ + 'id': '0000000000000000000000000000000000000000000000000000000000000000', + 'vout': 'ffffffff', + 'script_sig_size': '4d', + 'script_sig': '04ffff001d0104455468652054696d65732030332f4a616e' + '2f32303039204368616e63656c6c6f72206f6e206272696e' + '6b206f66207365636f6e64206261696c6f757420666f722062616e6b73', + 'sequence': 'ffffffff' + }], + 'outputs': [{ + 'value': '000000012a05f200', + 'script_pub_key_size': '43', + 'script_pub_key': '4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f' + '6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac' + }] + }] + } + + def test_parse_genesis_block(self): + blk_path = '/bitcoin-node/.bitcoin/blocks/blk00000.dat' + + with open(blk_path, 'rb') as f: + try: + block = deserialize_block(f) + self.assertEqual( + block, self.expected_genesis_block, 'Genesis block is not equal to expected genesis block' + ) + except InvalidMagicBytes as e: + self.fail(e) + + +class TestSearchImplementation(TestCase): + def test_search_block(self): + genesis_block_hash = '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f' + genesis_block = search_block_with(genesis_block_hash) + print('---- Genesis block ----') + print(genesis_block) + + first_block_hash = '00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048' + first_block = search_block_with(first_block_hash) + print('---- First block ----') + print(first_block)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/test_unit.py Wed Feb 02 21:16:10 2022 +0100 @@ -0,0 +1,64 @@ +from unittest import TestCase +from testfixtures import TempDirectory +from bitcaviar_plus.block import deserialize_block +from bitcaviar_plus.search import deserialize_block_search + + +class TestBlockUnit(TestCase): + """ + Test block.py file + """ + + genesis_block_hex = """ + f9beb4d91d0100000100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67 + 768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c0101000000010000000000000000000000000000000000000000 + 000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72 + 206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0 + fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6b + f11d5fac00000000 + """ + + def test_deserialize_header(self): + pass + + def test_deserialize_transaction_data(self): + pass + + +class TestHelpersUnit(TestCase): + """ + Test helpers.py file + """ + + def test_get_var_int(self): + pass + + def test_compute_hash(self): + pass + + +class TestSearchUnit(TestCase): + """ + Test search.py file + """ + + genesis_block_value = """ + 8be834000b0100080100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67 + 768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c + """ + + first_block_value = """ + 8be834011d0100812d08010000006fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000982051fd1e4ba744bbbe68 + 0e1fee14677ba1a3c3540bf7b1cdb606e857233e0e61bc6649ffff001d01e36299 + """ + + def test_deserialize_block_search(self): + with TempDirectory() as d: + genesis_block_search_binary = bytes.fromhex(self.genesis_block_value) + d.write('test_block_search.dat', genesis_block_search_binary) + + with open('{}/test_block_search.dat'.format(d.path), 'rb') as f: + deserialize_block_search(f) + + def test_create_file_with(self): + pass