GetTransactionsByBlockID is not working the same on testnet/mainnet and emulator

[bjartek]

Instructions

Problem

Calling this endpoint on emulator returns different data on emulator then what it does on mainnet or testnet. System transactions are not reported, even though they are executed. Scheduled callbacks are executed and evm transactions are sent but they are not reported the same way.

Steps to Reproduce

package main

import (
	"context"
	"fmt"

	"github.com/bjartek/overflow/v2"
	"github.com/onflow/flow-go-sdk"
)

func main() {


	// this has a normal flow transaction
	//	testnetBlockId := "dcda8ae35c30d2d4535f2150a9f792340dfc3fcbfcb23e1601b4202cd26b84a8"
	//o := overflow.Overflow(overflow.WithNetwork("testnet"))

	emulatorBlockId := "7bf09332ce45e96bc956122c7a198a7bf184e9ce6a4fd6f09c6c73b79af395fe"
	o := overflow.Overflow(overflow.WithExistingEmulator())

	tx, txR, err := o.Flowkit.GetTransactionsByBlockID(context.Background(), flow.HexToID(emulatorBlockId))
	if err != nil {
		panic(err)
	}

	for i, rawTx := range tx {

		t := rawTx
		tr := txR[i]

		fmt.Println("\n=== Transaction", i, "===")
		fmt.Println("ID:", t.ID().String())
		fmt.Println("ID(result):", tr.TransactionID.String())
		fmt.Println("BlockID:", tr.BlockID.String())
		fmt.Println("BlockHeight:", tr.BlockHeight)
		fmt.Println("CollectionID:", tr.CollectionID.String())
		fmt.Println("Script:")
		fmt.Println(string(t.Script))
		fmt.Println()

		fmt.Println("Events count:", len(tr.Events))
		for j, event := range tr.Events {
			fmt.Printf("  Event[%d]: %s (txIndex=%d, eventIndex=%d)\n", j, event.Type, event.TransactionIndex, event.EventIndex)
		}

	}
}

logs on emulator

=== Transaction 0 ===
ID: bca9d1cfefdcb0d843a5394dca9d4840f462705859c52a4e410509ed1fb04539
ID(result): bca9d1cfefdcb0d843a5394dca9d4840f462705859c52a4e410509ed1fb04539
BlockID: 7bf09332ce45e96bc956122c7a198a7bf184e9ce6a4fd6f09c6c73b79af395fe
BlockHeight: 13
CollectionID: 0000000000000000000000000000000000000000000000000000000000000000
Script:
import EVM from 0xf8d6e0586b0a20c7
import FungibleToken from 0xee82856bf20e2aa6
import FlowToken from 0x0ae53cb6e3f42a79

transaction(addr:String) {

    prepare(signer: auth(Storage) &Account) {
        let vaultRef = signer.storage.borrow<auth(FungibleToken.Withdraw) &FlowToken.Vault>(
            from: /storage/flowTokenVault
        ) ?? panic("Could not borrow reference to the owner's Vault!")

        let eoaAddressBytes: [UInt8; 20] = addr.decodeHex().toConstantSized<[UInt8; 20]>()!
        let eoaAddress: EVM.EVMAddress = EVM.EVMAddress(bytes: eoaAddressBytes)
        let fundVault <- vaultRef.withdraw(amount: 1.0) as! @FlowToken.Vault

        eoaAddress.deposit(from: <- fundVault)
    }

}


Events count: 7
  Event[0]: A.ee82856bf20e2aa6.FungibleToken.Withdrawn (txIndex=0, eventIndex=0)
  Event[1]: A.f8d6e0586b0a20c7.EVM.TransactionExecuted (txIndex=0, eventIndex=1)
  Event[2]: A.f8d6e0586b0a20c7.EVM.FLOWTokensDeposited (txIndex=0, eventIndex=2)
  Event[3]: A.ee82856bf20e2aa6.FungibleToken.Withdrawn (txIndex=0, eventIndex=3)
  Event[4]: A.0ae53cb6e3f42a79.FlowToken.TokensDeposited (txIndex=0, eventIndex=4)
  Event[5]: A.ee82856bf20e2aa6.FungibleToken.Deposited (txIndex=0, eventIndex=5)
  Event[6]: A.e5a8b7f23e8b548f.FlowFees.FeesDeducted (txIndex=0, eventIndex=6)

Why is the collection id 0000 here? This is not the same as is reported if i get the data from flow blocks get

Block ID		7bf09332ce45e96bc956122c7a198a7bf184e9ce6a4fd6f09c6c73b79af395fe
Parent ID		b609b44591b222a7fbd8ba771d750feec363d468105fc4dda9ab27bfdb0be41b
Proposal Timestamp	2025-10-17 08:59:07.208 +0000 UTC
Proposal Timestamp Unix	1760691547
Height			13
Status			Sealed
Total Seals		0
Total Collections	1
    Collection 0:	fdaeedac5fd76edc1ce52f58ed2f95d610017ce694d99fa6a335610a601dd244

Also there are no system transactions here.

Log on testnet for reference

=== Transaction 0 ===
ID: 87ea053b27db96d728d0f456d38f36e36ad4aaa5cdb8291ba25f3d13a2c24eec
ID(result): 87ea053b27db96d728d0f456d38f36e36ad4aaa5cdb8291ba25f3d13a2c24eec
BlockID: dcda8ae35c30d2d4535f2150a9f792340dfc3fcbfcb23e1601b4202cd26b84a8
BlockHeight: 285334204
CollectionID: 9ffbcee63049e13a4a840157c9cbb01030d9bf13cadf9cf7dbe2cfe781f27ded
Script:
import FlowToken from 0x7e60df042a9c0868
import FungibleToken from 0x9a0766d93b6608b7
import NonFungibleToken from 0x631e88ae7f1d7c20
import MetadataViews from 0x631e88ae7f1d7c20
import NFTCatalog from 0x324c34e1c517e4db
import NFTStorefrontV2 from 0x6225830c8c0957ba

transaction(saleItemPrice: UFix64) {
    let flowReceiver: Capability<&{FungibleToken.Receiver}>
    let collectionCap: Capability<auth(NonFungibleToken.Withdraw) &{NonFungibleToken.Provider, NonFungibleToken.Collection}>
    let storefront: auth(NFTStorefrontV2.CreateListing, NFTStorefrontV2.RemoveListing) &NFTStorefrontV2.Storefront
    var saleCuts: [NFTStorefrontV2.SaleCut]
    let saleItemID: UInt64
    let nftType: Type

    let collectionIdentifier: String
    let customID: String
    let commissionAmount: UFix64
    let expiry: UFix64

    prepare(acct: auth(Storage, Capabilities) &Account) {
        let seed = revertibleRandom<UInt64>()

        self.collectionIdentifier = "FlowtyTestNFT"
        self.customID = "flowty"
        self.commissionAmount = saleItemPrice * 0.025
        self.expiry = getCurrentBlock().timestamp + 7200.0 // two hours

        if acct.storage.borrow<&NFTStorefrontV2.Storefront>(from: NFTStorefrontV2.StorefrontStoragePath) == nil {
            let prev <- acct.storage.load<@AnyResource>(from: NFTStorefrontV2.StorefrontStoragePath)
            if prev != nil {
                acct.capabilities.unpublish(NFTStorefrontV2.StorefrontPublicPath)
            }
            destroy prev

            // Create a new empty Storefront
            let storefront <- NFTStorefrontV2.createStorefront()

            // save it to the account
            acct.storage.save(<-storefront, to: NFTStorefrontV2.StorefrontStoragePath)

            acct.capabilities.publish(
                acct.capabilities.storage.issue<&{NFTStorefrontV2.StorefrontPublic}>(NFTStorefrontV2.StorefrontStoragePath),
                at: NFTStorefrontV2.StorefrontPublicPath
            )
        }

        let value = NFTCatalog.getCatalogEntry(collectionIdentifier: self.collectionIdentifier) ?? panic("Provided collection is not in the NFT Catalog.")

        self.saleCuts = []

        // We need a provider capability, but one is not provided by default so we create one if needed.
        let nftCollectionProviderPrivatePath = PrivatePath(identifier: "nftCollectionProviderForNFTStorefront".concat(self.collectionIdentifier))!

        // Receiver for the sale cut.
        self.flowReceiver = acct.capabilities.get<&{FungibleToken.Receiver}>(/public/flowTokenReceiver)
        assert(self.flowReceiver.borrow() != nil, message: "Missing or mis-typed FlowToken receiver")

        var providerCap: Capability<auth(NonFungibleToken.Withdraw) &{NonFungibleToken.Provider, NonFungibleToken.Collection}>? = nil

        if providerCap == nil {
            providerCap = acct.capabilities.storage.issue<auth(NonFungibleToken.Withdraw) &{NonFungibleToken.Provider, NonFungibleToken.Collection}>(value.collectionData.storagePath)
        }
        self.collectionCap = providerCap!


        let collection = self.collectionCap.borrow<>()
            ?? panic("Could not borrow a reference to the collection")
        var totalRoyaltyCut = 0.0
        let effectiveSaleItemPrice = saleItemPrice - self.commissionAmount

        let c = acct.storage.borrow<&{NonFungibleToken.CollectionPublic}>(from: value.collectionData.storagePath) ?? panic("no collection setup")

        let ids = c.getIDs()
        let id = seed % UInt64(ids.length)
        self.saleItemID = ids[id]

        // Append the cut for the seller.
        self.saleCuts.append(NFTStorefrontV2.SaleCut(
            receiver: self.flowReceiver,
            amount: effectiveSaleItemPrice - totalRoyaltyCut
        ))
        assert(self.collectionCap.check(), message: "Missing or mis-typed NonFungibleToken.Provider, NonFungibleToken.Collection provider")

        self.storefront = acct.storage.borrow<auth(NFTStorefrontV2.CreateListing, NFTStorefrontV2.RemoveListing) &NFTStorefrontV2.Storefront>(from: NFTStorefrontV2.StorefrontStoragePath)
            ?? panic("Missing or mis-typed NFTStorefront Storefront")

        self.nftType = value.nftType
    }

    execute {
        // check for existing listings of the NFT
        var existingListingIDs = self.storefront.getExistingListingIDs(
            nftType: self.nftType,
            nftID: self.saleItemID
        )

        // remove existing listings
        for listingID in existingListingIDs {
            self.storefront.removeListing(listingResourceID: listingID)
        }

        // Create listing
        self.storefront.createListing(
            nftProviderCapability: self.collectionCap,
            nftType: self.nftType,
            nftID: self.saleItemID,
            salePaymentVaultType: Type<@FlowToken.Vault>(),
            saleCuts: self.saleCuts,
            marketplacesCapability: nil,
            customID: self.customID,
            commissionAmount: self.commissionAmount,
            expiry: UInt64(self.expiry)
        )
    }
}


Events count: 8
  Event[0]: flow.StorageCapabilityControllerIssued (txIndex=0, eventIndex=0)
  Event[1]: A.6225830c8c0957ba.NFTStorefrontV2.Listing.ResourceDestroyed (txIndex=0, eventIndex=1)
  Event[2]: A.6225830c8c0957ba.NFTStorefrontV2.ListingAvailable (txIndex=0, eventIndex=2)
  Event[3]: A.7e60df042a9c0868.FlowToken.TokensWithdrawn (txIndex=0, eventIndex=3)
  Event[4]: A.9a0766d93b6608b7.FungibleToken.Withdrawn (txIndex=0, eventIndex=4)
  Event[5]: A.7e60df042a9c0868.FlowToken.TokensDeposited (txIndex=0, eventIndex=5)
  Event[6]: A.9a0766d93b6608b7.FungibleToken.Deposited (txIndex=0, eventIndex=6)
  Event[7]: A.912d5440f7e3769e.FlowFees.FeesDeducted (txIndex=0, eventIndex=7)

=== Transaction 1 ===
ID: af35ecd8b485e41ed9fa68580557ced336cf46789e4c93af0378ea9812cc1a4b
ID(result): af35ecd8b485e41ed9fa68580557ced336cf46789e4c93af0378ea9812cc1a4b
BlockID: dcda8ae35c30d2d4535f2150a9f792340dfc3fcbfcb23e1601b4202cd26b84a8
BlockHeight: 285334204
CollectionID: 0000000000000000000000000000000000000000000000000000000000000000
Script:
import FlowTransactionScheduler from 0x8c5303eaa26202d6

// Process scheduled transactions by the FlowTransactionScheduler contract.
// This will be called by the FVM and all scheduled transactions that should be 
// executed will be processed. An event for each will be emitted.
transaction {
    prepare(serviceAccount: auth(BorrowValue) &Account) {
        let scheduler = serviceAccount.storage.borrow<auth(FlowTransactionScheduler.Process) &FlowTransactionScheduler.SharedScheduler>(from: FlowTransactionScheduler.storagePath)
            ?? panic("Could not borrow FlowTransactionScheduler")

        scheduler.process()
    }
}

Events count: 0

=== Transaction 2 ===
ID: dadf3e1bf916f6cb2510cbea00ed9be78cc1b7d2b9ec29f0ef1d469ead2dda2d
ID(result): dadf3e1bf916f6cb2510cbea00ed9be78cc1b7d2b9ec29f0ef1d469ead2dda2d
BlockID: dcda8ae35c30d2d4535f2150a9f792340dfc3fcbfcb23e1601b4202cd26b84a8
BlockHeight: 285334204
CollectionID: 0000000000000000000000000000000000000000000000000000000000000000
Script:
import FlowEpoch from 0x9eca2b38b18b5dfe
import NodeVersionBeacon from 0x8c5303eaa26202d6
import RandomBeaconHistory from 0x8c5303eaa26202d6
import EVM from 0x8c5303eaa26202d6
import Migration from 0x8c5303eaa26202d6

transaction {
    prepare(serviceAccount: auth(BorrowValue) &Account) {
        let epochHeartbeat = serviceAccount.storage.borrow<&FlowEpoch.Heartbeat>(from: FlowEpoch.heartbeatStoragePath)
            ?? panic("Could not borrow heartbeat from storage path")
        epochHeartbeat.advanceBlock()

        let versionBeaconHeartbeat = serviceAccount.storage
            .borrow<&NodeVersionBeacon.Heartbeat>(from: NodeVersionBeacon.HeartbeatStoragePath)
            ?? panic("Couldn't borrow NodeVersionBeacon.Heartbeat Resource")
        versionBeaconHeartbeat.heartbeat()

        let randomBeaconHistoryHeartbeat = serviceAccount.storage
            .borrow<&RandomBeaconHistory.Heartbeat>(from: RandomBeaconHistory.HeartbeatStoragePath)
            ?? panic("Couldn't borrow RandomBeaconHistory.Heartbeat Resource")
        randomBeaconHistoryHeartbeat.heartbeat(randomSourceHistory: randomSourceHistory())

        let evmHeartbeat = serviceAccount.storage
            .borrow<&EVM.Heartbeat>(from: /storage/EVMHeartbeat)
        evmHeartbeat?.heartbeat()

        let migrationAdmin = serviceAccount.storage
            .borrow<&Migration.Admin>(from: Migration.adminStoragePath)
        migrationAdmin?.migrate()
    }
}


Events count: 1
  Event[0]: A.8c5303eaa26202d6.EVM.BlockExecuted (txIndex=2, eventIndex=0)

Acceptance Criteria

It should be reported the same way in order to make good ux tools for both emulater and testnet.

Context

aether, forte hacks.
findlabs indexer for scheduled transactions.

[bjartek]

I dug a bit into this and it appears that the collectionID is not part of the storableResult for tranasctions even though it is part of the routine that commits a block.

Also in this pr #860 there are TODOs hanging that means that scheduled transactions are not handled correctly.

[bjartek]

The way i see this the emulator needs a new features that does

• stores the collectionID with transactions in the store
• records the system chunk transaction and the scheduled transaction system transaction in the store in a new dictionary.
• implements get-system command from the an spec

[jribbink]

I dug a bit into this and it appears that the collectionID is not part of the storableResult for tranasctions even though it is part of the routine that commits a block.

Also in this pr #860 there are TODOs hanging that means that scheduled transactions are not handled correctly.

Yeah sorry at the time that this support was added some of the APIs were in flux and the first priority was getting the feature out to early-adopter app developers who may not need full support. (IIRC they had just moved into the system TX space & it wasn't really clear where these results would live yet).

Thanks for bringing this issue up during developer office hours, and sorry for not noticing it earlier (things were quite busy in the lead up to Forte). I can take a look at it this week 🙏