feerates.py

import numpy as np
import matplotlib.pyplot as plt
import pickle

#STRUCTURE: (median, cumulative tx0 count, timestamp, 1000000, 5000000, 50000000)

def load_fees_raw(filename):
    """
    input: text file of fee information from bitcoincore
    output: list containing each line as a string
    """
    with open(filename) as fees_file:
        fees = fees_file.read().split("\n")
    return fees

def get_data(fees):
    """
    input: list of lines (output of load_fees_raw)
    output: dictionary with structure
        {block number:
            (median feerate,
            cumulative tx0 count,
            ntime,
            cumulative 1000000sat tx0 count,
            cumulative 5000000sat tx0 count,
            cumulative 50000000sat tx0 count
            )
        }
    """
    all_data = {}
    i = 0
    while i < len(fees):
        line = fees[i]
        if line[:28] != "all fees in block of height=":
            i += 1
            continue
        else:
            line = line.split(":")
            block_number = line[0].split("=")[1]
            block_fees = [int(fee) for fee in line[1].split() if fee != ""]
            data = []
            if block_fees:
                data.append(np.median(block_fees))
            else:
                data.append(None)
            if i < len(fees) and fees[i+1][:20] == "cumulative # of tx0s":
                block_time = fees[i+1].split('=')[2].split(' : ')[0]
                tx0s = fees[i+1].split('=')[2].split(' : ')[1]
                data.append(int(tx0s))
                data.append(block_time)
            i += 2
            denoms = [0,0,0]
            while i < len(fees) and fees[i][:28] != "all fees in block of height=":
                if fees[i][:14] == "denom: 1000000":
                    denoms[0] = int(fees[i].split(": ")[-1])
                if fees[i][:15] == "denom: 5000000 ":
                    denoms[1] = int(fees[i].split(": ")[-1])
                if fees[i][:15] == "denom: 50000000":
                    denoms[2] = int(fees[i].split(": ")[-1])
                i += 1
            data.extend(denoms)
            all_data[block_number] = tuple(data)

    return all_data


def split_given_size(a, size):
    """
    helper function for equal_epochs
    """
    return np.split(a, np.arange(size,len(a),size))


def equal_epochs(medians, tx0s, len_of_epoch=1440):
    """
    given (index-aligned) lists of median feerates and tx0 deltas between blocks,
    combine these into epochs of len_of_epoch blocks each. return average median
    feerates and tx0 deltas for each epoch
    """
    assert len(medians) == len(tx0s)
    feerates = []
    coinjoins = []
    medians = split_given_size(np.array(medians), len_of_epoch)
    tx0s = split_given_size(np.array(tx0s), len_of_epoch)
    feerates = [np.average(chunk) for chunk in medians]
    coinjoins = [np.sum(chunk) for chunk in tx0s]
    return (feerates, coinjoins)


def epochs_by_day(medians, tx0s, times, len_of_epoch=864000):
    """
    given (index-aligned) lists of median feerates, tx0 deltas between blocks,
    and ntime timestamps, combine these into epochs of len_of_epoch seconds each
    (one day is 86400 seconds). return average median feerates and tx0 deltas for
    each epoch
    """
    assert len(medians) == len(tx0s) and len(medians) == len(times)
    feerates = []
    coinjoins = []
    cur_fees = [medians[0]]
    cur_tx0s = [tx0s[0]]
    day = int(times[0]) // len_of_epoch
    i = 1
    while i < len(medians):
        while i < len(medians) and int(times[i]) // len_of_epoch == day:
            cur_fees.append(medians[i])
            cur_tx0s.append(tx0s[i])
            i += 1
        feerates.append(sum(cur_fees)/len(cur_fees))
        coinjoins.append(sum(cur_tx0s))
        cur_fees = []
        cur_tx0s = []
        if i < len(medians):
            day = int(times[i]) // len_of_epoch
    return feerates, coinjoins



if __name__ == "__main__":

    # pickle data as output by get_data for ease of use
    fees = load_fees_raw('fees-by-denomination-up-to-now.txt')
    all_data = get_data(fees)
    dbfile = open('data_pickle', 'ab')
    pickle.dump(all_data, dbfile)
    dbfile.close()

    medians = []
    tx0s = []
    times = []
    denom_1 = []
    denom_2 = []
    denom_3 = []

    # extract data from pickle in lists suitable for input into data processing functions
    dbfile = open('data_pickle', 'rb')
    db = pickle.load(dbfile)
    for keys in db:
        median, tx0, time, d1, d2, d3 = db[keys]
        if median is not None:
            medians.append(median)
        else:
            medians.append(0)
        tx0s.append(tx0)
        times.append(time)
        denom_1.append(d1)
        denom_2.append(d2)
        denom_3.append(d3)
    dbfile.close()

    # create tx0 delta lists per denomination
    tx0s_delta = [0]+[tx0s[i]-tx0s[i-1] for i in range(1, len(tx0s))]
    denom_1_delta = [0]+[denom_1[i]-denom_1[i-1] for i in range(1, len(denom_1))]
    denom_2_delta = [0]+[denom_2[i]-denom_2[i-1] for i in range(1, len(denom_2))]
    denom_3_delta = [0]+[denom_3[i]-denom_3[i-1] for i in range(1, len(denom_3))]

    # group into epochs (using total tx0s here, 864000 here is 10 days)
    feerates, coinjoins = epochs_by_day(medians, tx0s_delta, times, len_of_epoch=864000)


    # plot time series of tx0s and feerates
    x = np.arange(0, len(feerates))
    fig, ax1 = plt.subplots()
    ax1.plot(x, feerates, color='red', label='average median feerate', linewidth=0.5)
    ax1.set_xlabel('epoch number')
    ax1.set_ylabel('average median feerate', color='red')
    ax1.tick_params(axis='y', labelcolor='red')
    ax2 = ax1.twinx()
    ax2.plot(x, coinjoins, color='blue', label='# of tx0s', linewidth=0.5)
    ax2.set_ylabel('# of tx0s', color='blue')
    ax2.tick_params(axis='y', labelcolor='blue')
    fig.legend()

    # # plot scatterplot of feerates and tx0s
    # plt.scatter(feerates, coinjoins, s=10)
    # plt.xlabel("average median feerate per epoch (10 days)")
    # plt.ylabel("number of tx0s")

    # b, a = np.polyfit(feerates, coinjoins, deg=1)
    # xseq = np.linspace(0, 800, num=100)
    # plt.plot(xseq, a + b * xseq, color="k", lw=2.5)


    plt.show()