changeset 28:30535f42d0ff

refactor code
author Dennis Concepcion Martin <dennisconcepcionmartin@gmail.com>
date Wed, 02 Feb 2022 21:16:10 +0100
parents 348a07008703
children d9537541d623
files .github/workflows/python-publish.yml Dockerfile README.md src/bitcaviar_plus/block.py src/bitcaviar_plus/helpers.py src/bitcaviar_plus/search.py tests/implementation_testing.py tests/test_block.py tests/test_implementation.py tests/test_unit.py
diffstat 10 files changed, 239 insertions(+), 131 deletions(-) [+]
line wrap: on
line diff
--- a/.github/workflows/python-publish.yml	Thu Jan 06 12:10:25 2022 +0000
+++ b/.github/workflows/python-publish.yml	Wed Feb 02 21:16:10 2022 +0100
@@ -27,8 +27,8 @@
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Test
-      run: python -m unittest discover
+    - name: Testing units
+      run: python -m unittest tests.test_unit
     - name: Build package
       run: python -m build
     - name: Publish package
--- a/Dockerfile	Thu Jan 06 12:10:25 2022 +0000
+++ b/Dockerfile	Wed Feb 02 21:16:10 2022 +0100
@@ -13,4 +13,4 @@
 RUN ["python", "setup.py", "install"]
 
 # Test package
-CMD ["python3", "-u", "tests/implementation_testing.py"]
\ No newline at end of file
+CMD ["python", "-m", "unittest", "tests.test_implementation"]
\ No newline at end of file
--- a/README.md	Thu Jan 06 12:10:25 2022 +0000
+++ b/README.md	Wed Feb 02 21:16:10 2022 +0100
@@ -5,16 +5,27 @@
 
 
 # bitcaviar-plus
+
 A Bitcoin blockchain parser written in Python.
 
 ## Installation
-Clone repository
+
+### Recommended
+
 ```bash
 pip install bitcaviar-plus
 ```
 
+### Manual
+
+```bash
+python setup.py install
+```
+
 ## Usage
-### Deserialize first block from file `blk00000.dat`
+
+### Deserialize Genesis block
+
 ```python
 from bitcaviar_plus.block import deserialize_block
 
@@ -26,6 +37,7 @@
 ```
 
 ### Deserialize entire blockchain
+
 ```python
 import os
 from bitcaviar_plus.block import deserialize_block
@@ -47,6 +59,7 @@
 ```
 
 ### Example output
+
 ```json
 {
   "magic_number":"f9beb4d9",
--- a/src/bitcaviar_plus/block.py	Thu Jan 06 12:10:25 2022 +0000
+++ b/src/bitcaviar_plus/block.py	Wed Feb 02 21:16:10 2022 +0100
@@ -1,11 +1,10 @@
-
 """
 Deserialize methods for blocks
 """
 
 from bitcaviar_plus.block_structure import *
-from bitcaviar_plus.helpers import __get_var_int
-from bitcaviar_plus.helpers import __compute_hash
+from bitcaviar_plus.helpers import get_var_int
+from bitcaviar_plus.helpers import compute_hash
 from bitcaviar_plus.errors import InvalidMagicBytes
 
 
@@ -23,12 +22,12 @@
         raise InvalidMagicBytes(block.magic_number)
 
     block.size = f.read(4)[::-1].hex()
-    block_header, block.id = __deserialize_header(f)
-    block.transaction_count = __get_var_int(f)
+    block_header, block.id = deserialize_header(f)
+    block.transaction_count = get_var_int(f)
 
     transactions = []
     for transaction_number in range(int(block.transaction_count, 16)):
-        transactions.append(__deserialize_transaction_data(f))
+        transactions.append(deserialize_transaction_data(f))
 
     block_dict = block.__dict__
     block_dict['header'] = block_header
@@ -37,7 +36,9 @@
     return block_dict
 
 
-def __deserialize_header(f):
+# ---- SECONDARY METHODS ----
+
+def deserialize_header(f):
     """
     Deserialize block header
     More info: https://learnmeabitcoin.com/technical/block-header
@@ -48,7 +49,7 @@
     # Compute block hash
     before = f.tell()
     header = f.read(80)
-    block_hash = __compute_hash(header)
+    block_hash = compute_hash(header)
     f.seek(before)
 
     header = Header()
@@ -62,7 +63,7 @@
     return header.__dict__, block_hash
 
 
-def __deserialize_transaction_data(f):
+def deserialize_transaction_data(f):
     """
     Deserialize transaction data
     More info: https://learnmeabitcoin.com/technical/transaction-data
@@ -73,25 +74,25 @@
     transaction = Transaction()
     start_transaction_data = f.tell()
     transaction.version = f.read(4)[::-1].hex()
-    transaction.input_count = __get_var_int(f)
+    transaction.input_count = get_var_int(f)
 
     transaction_inputs = []
     for input_number in range(int(transaction.input_count, 16)):
         transaction_input = TransactionInput()
         transaction_input.id = f.read(32)[::-1].hex()
         transaction_input.vout = f.read(4)[::-1].hex()
-        transaction_input.script_sig_size = __get_var_int(f)
+        transaction_input.script_sig_size = get_var_int(f)
         transaction_input.script_sig = f.read(int(transaction_input.script_sig_size, 16)).hex()
         transaction_input.sequence = f.read(4)[::-1].hex()
         transaction_inputs.append(transaction_input.__dict__)
 
-    transaction.output_count = __get_var_int(f)
+    transaction.output_count = get_var_int(f)
 
     transaction_outputs = []
     for output_number in range(int(transaction.output_count, 16)):
         transaction_output = TransactionOutput()
         transaction_output.value = f.read(8)[::-1].hex()
-        transaction_output.script_pub_key_size = __get_var_int(f)
+        transaction_output.script_pub_key_size = get_var_int(f)
         transaction_output.script_pub_key = f.read(int(transaction_output.script_pub_key_size, 16)).hex()
         transaction_outputs.append(transaction_output.__dict__)
 
@@ -103,7 +104,7 @@
     f.seek(start_transaction_data)
     transaction_data = f.read(transaction_data_size)
     f.seek(end_transaction_data)
-    transaction.id = __compute_hash(transaction_data)
+    transaction.id = compute_hash(transaction_data)
 
     transaction_dict = transaction.__dict__
     transaction_dict['inputs'] = transaction_inputs
--- a/src/bitcaviar_plus/helpers.py	Thu Jan 06 12:10:25 2022 +0000
+++ b/src/bitcaviar_plus/helpers.py	Wed Feb 02 21:16:10 2022 +0100
@@ -5,7 +5,7 @@
 """
 
 
-def __get_var_int(f):
+def get_var_int(f):
     """
     A VarInt (variable integer) is a field used in transaction data to indicate the number of upcoming fields,
     or the length of an upcoming field.
@@ -28,7 +28,7 @@
     return number_of_transactions
 
 
-def __compute_hash(data):
+def compute_hash(data):
     """
     Get hash
     :param data: bytes, required
--- a/src/bitcaviar_plus/search.py	Thu Jan 06 12:10:25 2022 +0000
+++ b/src/bitcaviar_plus/search.py	Wed Feb 02 21:16:10 2022 +0100
@@ -1,4 +1,74 @@
-
 """
 Search methods for LEVELDB database
-"""
\ No newline at end of file
+"""
+
+try:
+    # noinspection PyUnresolvedReferences
+    import plyvel
+except ImportError:
+    # Avoid import error running unit tests
+    print("Couldn't import plyvel package. Are you running unit tests?")
+
+import tempfile
+
+
+def search_block_with(block_hash):
+    """
+    Search block with a given hash and get value
+    :param block_hash: string, required
+    :return: string
+    """
+
+    db = level_db_connect()
+    search_type = bytes.fromhex('62')  # 'b' (block) in hex is 62
+    block_hash = bytes.fromhex(block_hash)[::-1]
+    key = search_type + block_hash
+    value = db.get(key)
+    db.close()
+
+    return value.hex()
+
+
+# ---- SECONDARY METHODS ----
+
+def deserialize_block_search(f):
+    """
+    Deserialize value (block search)
+    More info: https://bitcoin.stackexchange.com/questions/67515/format-of-a-block-keys-contents-in-bitcoinds-leveldb
+    :param f: buffer, required
+    :return: dict
+    """
+
+    client_number = f.read(3)
+    print('Client number: {}'.format(client_number.hex()))
+    block_height = f.read(1)  # Var int 128?
+    print('Block height: {}'.format(block_height.hex()))
+    status = f.read(1)  # var int 128?
+    print('Status: {}'.format(status.hex()))
+    number_of_transactions = f.read(1)  # var int 128?
+    print('Number of transactions: {}'.format(number_of_transactions.hex()))
+
+
+def create_file_with(binary):
+    with tempfile.TemporaryFile() as fp:
+        fp.write(binary)
+        fp.seek(0)
+
+    return fp
+
+
+# noinspection PyUnresolvedReferences
+def level_db_connect():
+    db = plyvel.DB('/bitcoin-node/.bitcoin/blocks/index/', create_if_missing=False)
+
+    return db
+
+
+def get_128_var_int(f):
+    """
+    This var int is different from helpers.get_var_int
+    :param f:
+    :return: string
+    """
+
+    pass
--- a/tests/implementation_testing.py	Thu Jan 06 12:10:25 2022 +0000
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,46 +0,0 @@
-import os
-import plyvel
-import tempfile
-from bitcaviar_plus.block import deserialize_block
-from bitcaviar_plus.block import __deserialize_header
-from bitcaviar_plus.errors import InvalidMagicBytes
-
-
-def parse_genesis_block():
-    blk_path = '/bitcoin-node/.bitcoin/blocks/blk00355.dat'
-
-    with open(blk_path, 'rb') as f:
-        file_size = os.path.getsize(blk_path)
-        while f.tell() < file_size:
-            try:
-                block = deserialize_block(f)
-            except InvalidMagicBytes as e:
-                print(e)
-
-
-# noinspection PyUnresolvedReferences
-def iterate_leveldb_keys():
-    db = plyvel.DB('/bitcoin-node/.bitcoin/blocks/index/', create_if_missing=False)
-    for key, value in db:
-        print('---- RAW KEY ----')
-        print(key.hex())
-        print('---- LITTLE ENDIAN KEY ----')
-        print(key[::-1].hex())
-        print('---- RAW VALUE ----')
-        print(value[::-1].hex())
-        exit()
-
-
-# noinspection PyUnresolvedReferences
-def search_block():
-    db = plyvel.DB('/bitcoin-node/.bitcoin/blocks/index/', create_if_missing=False)
-    search_type = bytes.fromhex('62')  # 'b' (block) in hex is 62
-    block_hash = bytes.fromhex('000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f')[::-1]
-    key = search_type + block_hash
-    value = db.get(key)
-    print(value.hex())
-    db.close()
-
-
-if __name__ == '__main__':
-    search_block()
--- a/tests/test_block.py	Thu Jan 06 12:10:25 2022 +0000
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,62 +0,0 @@
-from unittest import TestCase
-from testfixtures import TempDirectory
-from bitcaviar_plus.block import deserialize_block
-
-
-class TestBlock(TestCase):
-    genesis_block_hex = """
-    f9beb4d91d0100000100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67
-    768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c0101000000010000000000000000000000000000000000000000
-    000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72
-    206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0
-    fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6b
-    f11d5fac00000000
-    """
-
-    expected_genesis_block = {
-        'magic_number': 'f9beb4d9',
-        'size': '0000011d',
-        'id': '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f',
-        'transaction_count': '01',
-        'header': {
-            'version': '00000001',
-            'previous_block_id': '0000000000000000000000000000000000000000000000000000000000000000',
-            'merkle_root': '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b',
-            'time': '495fab29',
-            'bits': '1d00ffff',
-            'nonce': '7c2bac1d'
-        },
-        'transactions': [{
-            'version': '00000001',
-            'input_count': '01',
-            'output_count': '01',
-            'lock_time': '00000000',
-            'id': '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b',
-            'inputs': [{
-                'id': '0000000000000000000000000000000000000000000000000000000000000000',
-                'vout': 'ffffffff',
-                'script_sig_size': '4d',
-                'script_sig': '04ffff001d0104455468652054696d65732030332f4a616e'
-                              '2f32303039204368616e63656c6c6f72206f6e206272696e'
-                              '6b206f66207365636f6e64206261696c6f757420666f722062616e6b73',
-                'sequence': 'ffffffff'
-            }],
-            'outputs': [{
-                'value': '000000012a05f200',
-                'script_pub_key_size': '43',
-                'script_pub_key': '4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f'
-                                  '6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac'
-            }]
-        }]
-    }
-
-    def test_deserialize_block(self):
-        with TempDirectory() as d:
-            test_block_binary = bytes.fromhex(self.genesis_block_hex)
-            d.write('test_block.dat', test_block_binary)
-
-            with open('{}/test_block.dat'.format(d.path), 'rb') as f:
-                block = deserialize_block(f)
-                self.assertEqual(
-                    block, self.expected_genesis_block, 'Genesis block is not equal to expected genesis block'
-                )
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test_implementation.py	Wed Feb 02 21:16:10 2022 +0100
@@ -0,0 +1,68 @@
+from unittest import TestCase
+from bitcaviar_plus.block import deserialize_block
+from bitcaviar_plus.errors import InvalidMagicBytes
+from bitcaviar_plus.search import search_block_with
+
+
+class TestBlockImplementation(TestCase):
+    expected_genesis_block = {
+        'magic_number': 'f9beb4d9',
+        'size': '0000011d',
+        'id': '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f',
+        'transaction_count': '01',
+        'header': {
+            'version': '00000001',
+            'previous_block_id': '0000000000000000000000000000000000000000000000000000000000000000',
+            'merkle_root': '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b',
+            'time': '495fab29',
+            'bits': '1d00ffff',
+            'nonce': '7c2bac1d'
+        },
+        'transactions': [{
+            'version': '00000001',
+            'input_count': '01',
+            'output_count': '01',
+            'lock_time': '00000000',
+            'id': '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b',
+            'inputs': [{
+                'id': '0000000000000000000000000000000000000000000000000000000000000000',
+                'vout': 'ffffffff',
+                'script_sig_size': '4d',
+                'script_sig': '04ffff001d0104455468652054696d65732030332f4a616e'
+                              '2f32303039204368616e63656c6c6f72206f6e206272696e'
+                              '6b206f66207365636f6e64206261696c6f757420666f722062616e6b73',
+                'sequence': 'ffffffff'
+            }],
+            'outputs': [{
+                'value': '000000012a05f200',
+                'script_pub_key_size': '43',
+                'script_pub_key': '4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f'
+                                  '6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac'
+            }]
+        }]
+    }
+
+    def test_parse_genesis_block(self):
+        blk_path = '/bitcoin-node/.bitcoin/blocks/blk00000.dat'
+
+        with open(blk_path, 'rb') as f:
+            try:
+                block = deserialize_block(f)
+                self.assertEqual(
+                    block, self.expected_genesis_block, 'Genesis block is not equal to expected genesis block'
+                )
+            except InvalidMagicBytes as e:
+                self.fail(e)
+
+
+class TestSearchImplementation(TestCase):
+    def test_search_block(self):
+        genesis_block_hash = '000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f'
+        genesis_block = search_block_with(genesis_block_hash)
+        print('---- Genesis block ----')
+        print(genesis_block)
+
+        first_block_hash = '00000000839a8e6886ab5951d76f411475428afc90947ee320161bbf18eb6048'
+        first_block = search_block_with(first_block_hash)
+        print('---- First block ----')
+        print(first_block)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/test_unit.py	Wed Feb 02 21:16:10 2022 +0100
@@ -0,0 +1,64 @@
+from unittest import TestCase
+from testfixtures import TempDirectory
+from bitcaviar_plus.block import deserialize_block
+from bitcaviar_plus.search import deserialize_block_search
+
+
+class TestBlockUnit(TestCase):
+    """
+    Test block.py file
+    """
+
+    genesis_block_hex = """
+    f9beb4d91d0100000100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67
+    768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c0101000000010000000000000000000000000000000000000000
+    000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72
+    206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0
+    fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6b
+    f11d5fac00000000
+    """
+
+    def test_deserialize_header(self):
+        pass
+
+    def test_deserialize_transaction_data(self):
+        pass
+
+
+class TestHelpersUnit(TestCase):
+    """
+    Test helpers.py file
+    """
+
+    def test_get_var_int(self):
+        pass
+
+    def test_compute_hash(self):
+        pass
+
+
+class TestSearchUnit(TestCase):
+    """
+    Test search.py file
+    """
+
+    genesis_block_value = """
+    8be834000b0100080100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67
+    768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c
+    """
+
+    first_block_value = """
+    8be834011d0100812d08010000006fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000982051fd1e4ba744bbbe68
+    0e1fee14677ba1a3c3540bf7b1cdb606e857233e0e61bc6649ffff001d01e36299
+    """
+
+    def test_deserialize_block_search(self):
+        with TempDirectory() as d:
+            genesis_block_search_binary = bytes.fromhex(self.genesis_block_value)
+            d.write('test_block_search.dat', genesis_block_search_binary)
+
+            with open('{}/test_block_search.dat'.format(d.path), 'rb') as f:
+                deserialize_block_search(f)
+
+    def test_create_file_with(self):
+        pass