blockchain-parser-2 0.1.6__py3-none-any.whl

blockchain_parser/__init__.py

@@ -0,0 +1,12 @@
+# Copyright (C) 2015-2016 The bitcoin-blockchain-parser developers
+#
+# This file is part of bitcoin-blockchain-parser.
+#
+# It is subject to the license terms in the LICENSE file found in the top-level
+# directory of this distribution.
+#
+# No part of bitcoin-blockchain-parser, including this file, may be copied,
+# modified, propagated, or distributed except according to the terms contained
+# in the LICENSE file.
+
+__version__ = "0.1.6"

blockchain_parser/address.py

@@ -0,0 +1,83 @@
+# Copyright (C) 2015-2016 The bitcoin-blockchain-parser developers
+#
+# This file is part of bitcoin-blockchain-parser.
+#
+# It is subject to the license terms in the LICENSE file found in the top-level
+# directory of this distribution.
+#
+# No part of bitcoin-blockchain-parser, including this file, may be copied,
+# modified, propagated, or distributed except according to the terms contained
+# in the LICENSE file.
+
+from bitcoin import base58
+from bitcoin.bech32 import CBech32Data
+from .utils import btc_ripemd160, double_sha256
+from .utils_taproot import from_taproot
+from binascii import b2a_hex
+
+
+class Address(object):
+    """Represents a bitcoin address"""
+
+    def __init__(self, hash, public_key, address, type, segwit_version):
+        self._hash = hash
+        self.public_key = public_key
+        self._address = address
+        self.type = type
+        self._segwit_version = segwit_version
+
+    def __repr__(self):
+        return "Address(addr=%s)" % self.address
+
+    @classmethod
+    def from_public_key(cls, public_key):
+        """Constructs an Address object from a public key"""
+        return cls(None, public_key, None, "normal", None)
+
+    @classmethod
+    def from_ripemd160(cls, hash, type="normal"):
+        """Constructs an Address object from a RIPEMD-160 hash, it may be a
+        normal address or a P2SH address, the latter is indicated by setting
+        type to 'p2sh'"""
+        return cls(hash, None, None, type, None)
+
+    @classmethod
+    def from_bech32(cls, hash, segwit_version):
+        """Constructs an Address object from a bech32 hash."""
+        return cls(hash, None, None, "bech32", segwit_version)
+
+    @classmethod
+    def from_bech32m(cls, hash, segwit_version):
+        """Constructs an Address object from a bech32m script."""
+        return cls(hash, None, None, "bech32m", segwit_version)
+
+    @property
+    def hash(self):
+        """Returns the RIPEMD-160 hash corresponding to this address"""
+        if self.public_key is not None and self._hash is None:
+            self._hash = btc_ripemd160(self.public_key)
+        return self._hash
+
+    @property
+    def address(self):
+        """Returns the encoded representation of this address.
+        If Taproot, it's encoded using bech32m,
+        if SegWit, it's encoded using bech32,
+        otherwise using base58
+        """
+        if self._address is None:
+            if self.type == "bech32m":
+                tweaked_pubkey = b2a_hex(self.hash).decode("ascii")
+                self._address = from_taproot(tweaked_pubkey)
+            elif self.type != "bech32":
+                version = b'\x00' if self.type == "normal" else b'\x05'
+                checksum = double_sha256(version + self.hash)
+
+                self._address = base58.encode(version + self.hash + checksum[:4])
+            else:
+                bech_encoded = CBech32Data.from_bytes(self._segwit_version, self._hash)
+                self._address = str(bech_encoded)
+        return self._address
+
+    def is_p2sh(self):
+        return self.type == "p2sh"

blockchain_parser/block.py

@@ -0,0 +1,104 @@
+# Copyright (C) 2015-2016 The bitcoin-blockchain-parser developers
+#
+# This file is part of bitcoin-blockchain-parser.
+#
+# It is subject to the license terms in the LICENSE file found in the top-level
+# directory of this distribution.
+#
+# No part of bitcoin-blockchain-parser, including this file, may be copied,
+# modified, propagated, or distributed except according to the terms contained
+# in the LICENSE file.
+
+from .transaction import Transaction
+from .block_header import BlockHeader
+from .utils import format_hash, decode_compactsize, double_sha256
+
+
+def get_block_transactions(raw_hex):
+    """Given the raw hexadecimal representation of a block,
+    yields the block's transactions
+    """
+    # Skipping the header
+    transaction_data = raw_hex[80:]
+
+    # Decoding the number of transactions, offset is the size of
+    # the varint (1 to 9 bytes)
+    n_transactions, offset = decode_compactsize(transaction_data)
+
+    for i in range(n_transactions):
+        transaction = None
+        # Try from 1024 (1KiB) -> 512MiB slice widths
+        for j in range(0, 20):
+            try:
+                offset_e = offset + (1024 * 2 ** j)
+                transaction = Transaction.from_hex(
+                    transaction_data[offset:offset_e])
+                break
+            except Exception:
+                transaction = None
+        if transaction is None:
+            raise ValueError(
+                "failed to parse transaction %d at offset %d" % (i, offset)
+            )
+
+        yield transaction
+        # Skipping to the next transaction
+        offset += transaction.size
+
+
+class Block(object):
+    """
+    Represents a Bitcoin block, contains its header and its transactions.
+    """
+
+    def __init__(self, raw_hex, height=None, blk_file=None):
+        self.hex = raw_hex
+        self._hash = None
+        self._transactions = None
+        self._header = None
+        self._n_transactions = None
+        self.size = len(raw_hex)
+        self.height = height
+        self.blk_file = blk_file
+
+    def __repr__(self):
+        return "Block(%s)" % self.hash
+
+    @classmethod
+    def from_hex(cls, raw_hex):
+        """Builds a block object from its bytes representation"""
+        return cls(raw_hex)
+
+    @property
+    def hash(self):
+        """Returns the block's hash (double sha256 of its 80 bytes header"""
+        if self._hash is None:
+            self._hash = format_hash(double_sha256(self.hex[:80]))
+        return self._hash
+
+    @property
+    def n_transactions(self):
+        """Return the number of transactions contained in this block,
+        it is faster to use this than to use len(block.transactions)
+        as there's no need to parse all transactions to get this information
+        """
+        if self._n_transactions is None:
+            self._n_transactions = decode_compactsize(self.hex[80:])[0]
+
+        return self._n_transactions
+
+    @property
+    def transactions(self):
+        """Returns a list of the block's transactions represented
+        as Transaction objects"""
+        if self._transactions is None:
+            self._transactions = list(get_block_transactions(self.hex))
+
+        return self._transactions
+
+    @property
+    def header(self):
+        """Returns a BlockHeader object corresponding to this block"""
+        if self._header is None:
+            self._header = BlockHeader.from_hex(self.hex[:80])
+        return self._header

blockchain_parser/block_header.py

@@ -0,0 +1,90 @@
+# Copyright (C) 2015-2016 The bitcoin-blockchain-parser developers
+#
+# This file is part of bitcoin-blockchain-parser.
+#
+# It is subject to the license terms in the LICENSE file found in the top-level
+# directory of this distribution.
+#
+# No part of bitcoin-blockchain-parser, including this file, may be copied,
+# modified, propagated, or distributed except according to the terms contained
+# in the LICENSE file.
+
+from datetime import datetime, timezone
+from bitcoin.core import CBlockHeader
+
+from .utils import decode_uint32, format_hash
+
+
+class BlockHeader(object):
+    """Represents a block header"""
+
+    def __init__(self, raw_hex):
+        self._version = None
+        self._previous_block_hash = None
+        self._merkle_root = None
+        self._timestamp = None
+        self._bits = None
+        self._nonce = None
+        self._difficulty = None
+
+        self.hex = raw_hex[:80]
+
+    def __repr__(self):
+        return "BlockHeader(previous_block_hash=%s)" % self.previous_block_hash
+
+    @classmethod
+    def from_hex(cls, raw_hex):
+        """Builds a BlockHeader object from its bytes representation"""
+        return cls(raw_hex)
+
+    @property
+    def version(self):
+        """Return the block's version"""
+        if self._version is None:
+            self._version = decode_uint32(self.hex[:4])
+        return self._version
+
+    @property
+    def previous_block_hash(self):
+        """Return the hash of the previous block"""
+        if self._previous_block_hash is None:
+            self._previous_block_hash = format_hash(self.hex[4:36])
+        return self._previous_block_hash
+
+    @property
+    def merkle_root(self):
+        """Returns the block's merkle root"""
+        if self._merkle_root is None:
+            self._merkle_root = format_hash(self.hex[36:68])
+        return self._merkle_root
+
+    @property
+    def timestamp(self):
+        """Returns the timestamp of the block as a UTC datetime object"""
+        if self._timestamp is None:
+            self._timestamp = datetime.fromtimestamp(
+                decode_uint32(self.hex[68:72]), timezone.utc
+            )
+        return self._timestamp
+
+    @property
+    def bits(self):
+        """Returns the bits (difficulty target) of the block"""
+        if self._bits is None:
+            self._bits = decode_uint32(self.hex[72:76])
+        return self._bits
+
+    @property
+    def nonce(self):
+        """Returns the block's nonce"""
+        if self._nonce is None:
+            self._nonce = decode_uint32(self.hex[76:80])
+        return self._nonce
+
+    @property
+    def difficulty(self):
+        """Returns the block's difficulty target as a float"""
+        if self._difficulty is None:
+            self._difficulty = CBlockHeader.calc_difficulty(self.bits)
+
+        return self._difficulty

blockchain_parser/blockchain.py

@@ -0,0 +1,302 @@
+# Copyright (C) 2015-2016 The bitcoin-blockchain-parser developers
+#
+# This file is part of bitcoin-blockchain-parser.
+#
+# It is subject to the license terms in the LICENSE file found in the top-level
+# directory of this distribution.
+#
+# No part of bitcoin-blockchain-parser, including this file, may be copied,
+# modified, propagated, or distributed except according to the terms contained
+# in the LICENSE file.
+
+import os
+import mmap
+import struct
+import pickle
+import stat
+
+from blockchain_parser.transaction import Transaction
+from blockchain_parser.index import DBTransactionIndex
+from blockchain_parser import utils
+from binascii import unhexlify
+from binascii import hexlify
+from .block import Block
+from .index import DBBlockIndex
+from .utils import format_hash
+from .block_header import BlockHeader
+
+
+# Constant separating blocks in the .blk files
+BITCOIN_CONSTANT = b"\xf9\xbe\xb4\xd9"
+
+
+def get_files(path):
+    """
+    Given the path to the .bitcoin directory, returns the sorted list of .blk
+    files contained in that directory
+    """
+    if not stat.S_ISDIR(os.stat(path)[stat.ST_MODE]):
+        return [path]
+    files = os.listdir(path)
+    files = [f for f in files if f.startswith("blk") and f.endswith(".dat")]
+    files = map(lambda x: os.path.join(path, x), files)
+    return sorted(files)
+
+def get_undo_files(path):
+    """
+    Given the path to the .bitcoin directory, returns the sorted list of rev*.dat
+    files contained in that directory
+    """
+    if not stat.S_ISDIR(os.stat(path)[stat.ST_MODE]):
+        return [path]
+    files = os.listdir(path)
+    files = [f for f in files if f.startswith("rev") and f.endswith(".dat")]
+    files = map(lambda x: os.path.join(path, x), files)
+    return sorted(files)
+
+
+def get_blocks(blockfile):
+    """
+    Given the name of a .dat file, for every block contained in the file,
+    yields its raw hexadecimal value
+    """
+    with open(blockfile, "rb") as f:
+        if os.name == 'nt':
+            size = os.path.getsize(f.name)
+            raw_data = mmap.mmap(f.fileno(), size, access=mmap.ACCESS_READ)
+        else:
+            # Unix-only call, will not work on Windows, see python doc.
+            raw_data = mmap.mmap(f.fileno(), 0, prot=mmap.PROT_READ)
+        length = len(raw_data)
+        offset = 0
+        block_count = 0
+        while offset < (length - 4):
+            if raw_data[offset:offset+4] == BITCOIN_CONSTANT:
+                offset += 4
+                size = struct.unpack("<I", raw_data[offset:offset+4])[0]
+                offset += 4 + size
+                block_count += 1
+                yield raw_data[offset-size:offset]
+            else:
+                offset += 1
+        raw_data.close()
+
+
+def get_block(blockfile, offset):
+    """Extracts a single block from the blockfile at the given offset"""
+    with open(blockfile, "rb") as f:
+        f.seek(offset - 4)  # Size is present 4 bytes before the db offset
+        size, = struct.unpack("<I", f.read(4))
+        return f.read(size)
+
+
+def _chain_work(bits):
+    """Return the proof-of-work contributed by a compact nBits target."""
+    if not bits:
+        return 0
+    exponent = bits >> 24
+    mantissa = bits & 0xffffff
+    if mantissa == 0:
+        return 0
+    if exponent <= 3:
+        target = mantissa >> (8 * (3 - exponent))
+    else:
+        target = mantissa << (8 * (exponent - 3))
+    if target <= 0:
+        return 0
+    return (1 << 256) // (target + 1)
+
+
+def select_main_chain(block_indexes):
+    """Return the most-work chain, oldest block first.
+
+    LevelDB stores block index entries under their hash, so iteration order
+    is not block height. Forks are resolved by total chain work, the same
+    rule Bitcoin Core uses to choose the active chain.
+    """
+    if not block_indexes:
+        return []
+
+    by_hash = {}
+    for block in block_indexes:
+        by_hash[block.hash] = block
+
+    has_successor = {block.prev_hash for block in block_indexes}
+    tips = [block for block in block_indexes if block.hash not in has_successor]
+    if not tips:
+        tips = list(block_indexes)
+
+    memo = {}
+
+    def work_of(start_block):
+        stack = [start_block]
+        stacked = {start_block.hash}
+        while stack:
+            current = stack[-1]
+            if current.hash in memo:
+                stack.pop()
+                stacked.discard(current.hash)
+                continue
+            prev = by_hash.get(current.prev_hash)
+            if prev is not None and prev.hash not in memo:
+                if prev.hash in stacked:
+                    prev = None
+                else:
+                    stack.append(prev)
+                    stacked.add(prev.hash)
+                    continue
+            prev_work = memo[prev.hash] if prev is not None else 0
+            memo[current.hash] = prev_work + _chain_work(
+                getattr(current, "bits", None)
+            )
+            stack.pop()
+            stacked.discard(current.hash)
+        return memo[start_block.hash]
+
+    for tip in tips:
+        work_of(tip)
+
+    best = max(
+        tips,
+        key=lambda block: (memo[block.hash], block.height, block.hash),
+    )
+
+    ordered = []
+    seen = set()
+    current = best
+    while current is not None and current.hash not in seen:
+        seen.add(current.hash)
+        ordered.append(current)
+        current = by_hash.get(current.prev_hash)
+    ordered.reverse()
+    return ordered
+
+
+def iter_chain_indexes(ordered, start=0, end=None):
+    """Select main-chain index entries for a height window.
+
+    When ``end`` is greater than ``start``, heights ``start`` inclusive
+    through ``end`` exclusive are returned. When ``start`` is greater than
+    ``end``, heights from ``start`` down to ``end`` are returned and both
+    ends are included. Entries without block data are omitted.
+    """
+    if not ordered:
+        return []
+
+    by_height = {}
+    for block in ordered:
+        by_height[block.height] = block
+
+    if end is None:
+        end = max(by_height) + 1
+
+    if end < start:
+        heights = range(start, end - 1, -1)
+    else:
+        heights = range(start, end)
+
+    selected = []
+    for height in heights:
+        block = by_height.get(height)
+        if block is None:
+            continue
+        if getattr(block, "file", -1) == -1 or getattr(block, "data_pos", -1) == -1:
+            continue
+        selected.append(block)
+    return selected
+
+
+def _load_block_indexes(index_path, cache):
+    """Load block index entries from a pickle cache or from LevelDB."""
+    block_indexes = None
+    if cache and os.path.exists(cache):
+        with open(cache, 'rb') as f:
+            block_indexes = pickle.load(f)
+        if not block_indexes or not hasattr(block_indexes[0], "bits"):
+            block_indexes = None
+
+    if block_indexes is None:
+        import plyvel
+        with plyvel.DB(index_path, compression=None) as db:
+            # Block index entries are stored with keys prefixed by 'b'
+            with db.iterator(prefix=b'b') as iterator:
+                block_indexes = [
+                    DBBlockIndex(format_hash(k[1:]), v) for k, v in iterator
+                ]
+        if cache:
+            with open(cache, 'wb') as f:
+                pickle.dump(block_indexes, f)
+    return block_indexes
+
+
+class Blockchain(object):
+    """Represent the blockchain contained in the series of .blk files
+    maintained by bitcoind.
+    """
+
+    def __init__(self, path):
+        self.path = path
+
+    def get_unordered_blocks(self):
+        """Yields the blocks contained in the .blk files as is,
+        without ordering them according to height.
+        """
+        for blk_file in get_files(self.path):
+            for raw_block in get_blocks(blk_file):
+                yield Block(raw_block, None, os.path.split(blk_file)[1])
+
+    def get_ordered_blocks(self, index, start=0, end=None, cache=None):
+        """Yield main-chain blocks in height order.
+
+        ``index`` is the path of Bitcoin Core's LevelDB block index. ``start``
+        and ``end`` are block heights. When ``end`` is greater than ``start``,
+        heights ``start`` inclusive through ``end`` exclusive are yielded.
+        When ``start`` is greater than ``end``, blocks are yielded from
+        ``start`` down to ``end`` and both heights are included.
+
+        ``cache`` stores the parsed index with pickle. Delete that file to
+        pick up blocks written after the cache was created. Header-only and
+        pruned entries are skipped.
+        """
+        block_indexes = _load_block_indexes(index, cache)
+        ordered = select_main_chain(block_indexes)
+        for blk_idx in iter_chain_indexes(ordered, start, end):
+            blk_file = os.path.join(self.path, "blk%05d.dat" % blk_idx.file)
+            yield Block(get_block(blk_file, blk_idx.data_pos), blk_idx.height)
+
+    def get_transaction(self, txid, db):
+        """Yields the transaction contained in the .blk files as a python
+         object, similar to
+         https://developer.bitcoin.org/reference/rpc/getrawtransaction.html
+        """
+
+        byte_arr = bytearray.fromhex(txid)
+        byte_arr.reverse()
+        tx_hash = hexlify(b't').decode('utf-8') + \
+            hexlify(byte_arr).decode('utf-8')
+
+        tx_hash_fmtd = unhexlify(tx_hash)
+        raw_hex = db.get(tx_hash_fmtd)
+        if raw_hex is None:
+            return None
+
+        tx_idx = DBTransactionIndex(utils.format_hash(tx_hash_fmtd), raw_hex)
+        blk_file = os.path.join(self.path, "blk%05d.dat" % tx_idx.blockfile_no)
+        raw_hex = get_block(blk_file, tx_idx.file_offset)
+
+        offset = tx_idx.block_offset
+
+        transaction_data = raw_hex[80:]
+        block_header_data = raw_hex[:80]
+        # Try from 1024 (1KiB) -> 1073741824 (1GiB) slice widths
+        for j in range(0, 20):
+            try:
+                block_header = BlockHeader.from_hex(block_header_data)
+                offset_e = offset + (1024 * 2 ** j)
+                transaction = Transaction.from_hex(
+                    transaction_data[offset:offset_e])
+                return [block_header, transaction]
+            except Exception:
+                continue
+
+        return None

blockchain_parser/index.py

@@ -0,0 +1,81 @@
+from struct import unpack
+
+from .utils import format_hash
+
+BLOCK_HAVE_DATA = 8
+BLOCK_HAVE_UNDO = 16
+
+
+def _read_varint(raw_hex):
+    """
+    Reads the weird format of VarInt present in src/serialize.h of bitcoin core
+    and being used for storing data in the leveldb.
+    This is not the VARINT format described for general bitcoin serialization
+    use.
+    """
+    n = 0
+    pos = 0
+    while True:
+        data = raw_hex[pos]
+        pos += 1
+        n = (n << 7) | (data & 0x7f)
+        if data & 0x80 == 0:
+            return n, pos
+        n += 1
+
+
+class DBBlockIndex(object):
+    def __init__(self, blk_hash, raw_hex):
+        self.hash = blk_hash
+        pos = 0
+        n_version, i = _read_varint(raw_hex[pos:])
+        pos += i
+        self.height, i = _read_varint(raw_hex[pos:])
+        pos += i
+        self.status, i = _read_varint(raw_hex[pos:])
+        pos += i
+        self.n_tx, i = _read_varint(raw_hex[pos:])
+        pos += i
+        if self.status & (BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO):
+            self.file, i = _read_varint(raw_hex[pos:])
+            pos += i
+        else:
+            self.file = -1
+
+        if self.status & BLOCK_HAVE_DATA:
+            self.data_pos, i = _read_varint(raw_hex[pos:])
+            pos += i
+        else:
+            self.data_pos = -1
+        if self.status & BLOCK_HAVE_UNDO:
+            self.undo_pos, i = _read_varint(raw_hex[pos:])
+            pos += i
+
+        assert (pos + 80 == len(raw_hex))
+        self.version, p, m, self.time, self.bits, self.nonce = unpack(
+            "<I32s32sIII",
+            raw_hex[-80:]
+        )
+        self.prev_hash = format_hash(p)
+        self.merkle_root = format_hash(m)
+
+    def __repr__(self):
+        return "DBBlockIndex(%s, height=%d, file_no=%d, file_pos=%d)" \
+               % (self.hash, self.height, self.file, self.data_pos)
+
+
+class DBTransactionIndex(object):
+    def __init__(self, txn_hash, raw_hex):
+        self.hash = txn_hash
+        pos = 0
+        self.blockfile_no, i = _read_varint(raw_hex[pos:])
+        pos += i
+        self.file_offset, i = _read_varint(raw_hex[pos:])
+        pos += i
+        self.block_offset, i = _read_varint(raw_hex[pos:])
+
+    def __repr__(self):
+        return "DBTransactionIndex(%s, blockfile_no=%d, " \
+               "file_offset=%d, block_offset=%d)" \
+               % (self.hash, self.blockfile_no,
+                  self.file_offset, self.block_offset)