Skip to content

Commit ff2802b

Browse files
tdrztdrz
authored
Prepopulated FS (#935)
1 parent 2ae666f commit ff2802b

File tree

16 files changed

+333
-2
lines changed

16 files changed

+333
-2
lines changed

‎.changeset/old-gorillas-allow.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@electric-sql/pglite-prepopulatedfs': patch
3+
---
4+
5+
New package prepopulatedfs providing an already inited fs to skip initdb, which leads to shorted startup times.

‎.github/workflows/build_and_test.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,9 @@ jobs:
231231
- name: Install dependencies
232232
run: pnpm install --frozen-lockfile
233233

234+
- name: Build pglite-utils
235+
run: pnpm --filter "@electric-sql/pglite-utils" build
236+
234237
- name: Build pg-protocol
235238
run: pnpm --filter "@electric-sql/pg-protocol" build
236239

‎docs/.vitepress/config.mts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,8 @@ export default defineConfig({
127127
{ text: 'pgdump', link: '/pglite-tools#pgDump' },
128128
],
129129
},
130-
{ text: 'Upgrade between minor versions', link: '/docs/upgrade' },
130+
{ text: 'Upgrade path', link: '/docs/upgrade' },
131+
{ text: 'Prepopulated FS', link: '/docs/prepopulatedfs' },
131132
],
132133
},
133134
{

‎docs/docs/prepopulatedfs.md‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# Pre-populated FS
2+
3+
A pre-populated FS that you can use instead of letting initdb run (which is the default). This can lead to faster startup times because initdb doesn't need to run.
4+
5+
This package contains an archive as a static asset that you can access through the `dataDir()` function.
6+
7+
:::info
8+
The prepopulated FS is created during build on our CI and therefore guaranteed to work only for the corresponding version of PGlite from which it was created. If you encounter issues, make sure this package is up to date with your PGlite version.
9+
:::
10+
11+
## Installation
12+
13+
```bash
14+
npm install @electric-sql/pglite-prepopulatedfs
15+
# or
16+
yarn add @electric-sql/pglite-prepopulatedfs
17+
# or
18+
pnpm add @electric-sql/pglite-prepopulatedfs
19+
```
20+
21+
## Usage
22+
23+
```typescript
24+
import { PGlite } from '@electric-sql/pglite'
25+
import { dataDir } from '@electric-sql/pglite-prepopulatedfs'
26+
27+
// Create a PGlite instance with the prepopulated FS
28+
const pg = await PGlite.create({
29+
loadDataDir: await dataDir(),
30+
})
31+
```
32+
33+
As an example, this is useful when you have multiple test, each with its own PGlite instance. Consider the following usage with vitest:
34+
35+
```typescript
36+
import { describe, it, expect, beforeEach } from 'vitest'
37+
import { PGlite } from '@electric-sql/pglite'
38+
import { dataDir } from '@electric-sql/pglite-prepopulatedfs'
39+
40+
describe('query and exec with different data sizes', () => {
41+
let pg: PGlite
42+
43+
beforeEach(async () => {
44+
pg = await PGlite.create({
45+
loadDataDir: await dataDir(),
46+
})
47+
48+
await pg.exec(`
49+
// setup default data
50+
`)
51+
})
52+
53+
describe('test no. 1', () => {
54+
...
55+
})
56+
57+
describe('test no. 2', () => {
58+
...
59+
})
60+
61+
// many more tests here
62+
})
63+
```
64+
65+
Although more bandwidth is needed to download the `@electric-sql/pglite-prepopulatedfs` package, the tests will run faster as PGlite doesn't need to run `initdb` for each one of them.
66+
67+
The same applies if your application needs to instantiate PGlite over and over again with a clean slate. You will initialy use more bandwidth but will save on speed in the long-run.
68+
69+
## Benchmarking
70+
71+
A simple benchmarking is done as part of our automated testing in `packages/pglite-prepopulatedfs/tests/prepopulatedfs.test.ts`.
72+
73+
Here is a sample output on an Apple M1:
74+
75+
```
76+
initdb duration: prepopulated avg (trimmed) 263.38 ms vs. classic initdb 886.29 ms.
77+
Speedup: 3.37x
78+
```

‎packages/pglite-prepopulatedfs/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
release/

‎packages/pglite-prepopulatedfs/README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# pglite-prepopulatedfs
2+
3+
A prepopulated FS so no initdb is running on startup.
4+
5+
Install with:
6+
7+
```bash
8+
npm install @electric-sql/pglite-prepopulatedfs
9+
```

‎packages/pglite-prepopulatedfs/eslint.config.js‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import globals from 'globals'
2+
import rootConfig from '../../eslint.config.js'
3+
4+
export default [
5+
...rootConfig,
6+
{
7+
ignores: ['release/**/*', 'examples/**/*', 'dist/**/*'],
8+
},
9+
{
10+
languageOptions: {
11+
globals: {
12+
...globals.browser,
13+
...globals.node,
14+
},
15+
},
16+
rules: {
17+
...rootConfig.rules,
18+
'@typescript-eslint/no-explicit-any': 'off',
19+
},
20+
},
21+
{
22+
files: ['tests/targets/deno/**/*.js'],
23+
languageOptions: {
24+
globals: {
25+
Deno: false,
26+
},
27+
},
28+
},
29+
]

‎packages/pglite-prepopulatedfs/package.json‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
{
2+
"name": "@electric-sql/pglite-prepopulatedfs",
3+
"version": "0.0.1",
4+
"description": "Prepopulated filesystem for faster PGlite startups",
5+
"author": "Electric DB Limited",
6+
"homepage": "https://pglite.dev",
7+
"license": "Apache-2.0",
8+
"repository": {
9+
"type": "git",
10+
"url": "git+https://github.com/electric-sql/pglite",
11+
"directory": "packages/pglite-prepopulatedfs"
12+
},
13+
"keywords": [
14+
"postgres",
15+
"sql",
16+
"database",
17+
"wasm",
18+
"pglite",
19+
"initdb"
20+
],
21+
"private": false,
22+
"publishConfig": {
23+
"access": "public"
24+
},
25+
"files": ["./dist"],
26+
"type": "module",
27+
"types": "dist/index.d.ts",
28+
"main": "dist/index.cjs",
29+
"module": "dist/index.js",
30+
"exports": {
31+
".": {
32+
"import": {
33+
"types": "./dist/index.d.ts",
34+
"default": "./dist/index.js"
35+
},
36+
"require": {
37+
"types": "./dist/index.d.cts",
38+
"default": "./dist/index.cjs"
39+
}
40+
}
41+
},
42+
"scripts": {
43+
"build": "tsx ./scripts/generateFS.ts && tsup",
44+
"check:exports": "attw . --pack --profile node16",
45+
"lint": "eslint ./tests --report-unused-disable-directives --max-warnings 0",
46+
"format": "prettier --write ./tests",
47+
"typecheck": "tsc",
48+
"stylecheck": "pnpm lint && prettier --check ./tests",
49+
"test": "vitest",
50+
"prepublishOnly": "pnpm check:exports"
51+
},
52+
"devDependencies": {
53+
"@arethetypeswrong/cli": "^0.18.1",
54+
"@electric-sql/pglite": "workspace:*",
55+
"@types/emscripten": "^1.41.1",
56+
"@types/node": "^20.16.11",
57+
"tsx": "^4.19.2",
58+
"vitest": "^1.3.1",
59+
"@electric-sql/pglite-utils": "workspace:*"
60+
}
61+
}

‎packages/pglite-prepopulatedfs/scripts/bundle-static-assets.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
import { copyFiles, findAndReplaceInDir } from '@electric-sql/pglite-utils/scripts/fileUtils'
2+
3+
export async function doBundle() {
4+
await copyFiles('./release', './dist')
5+
await findAndReplaceInDir('./dist', /\.\.\/release\//g, './', ['.js', '.cjs', '.map'])
6+
}
7+
8+
await doBundle()

‎packages/pglite-prepopulatedfs/scripts/generateFS.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import { PGlite } from '@electric-sql/pglite'
2+
import { resolve } from 'path'
3+
const fs = await import('fs')
4+
5+
const pglite = await PGlite.create()
6+
const dataDirArchive = await pglite.dumpDataDir('gzip')
7+
console.info('Removing release folder')
8+
fs.rmSync(resolve('release'), { recursive: true, force: true })
9+
try {
10+
console.info('Creating release folder')
11+
fs.mkdirSync(resolve('release'))
12+
console.info('Writing preloaded FS file to disk')
13+
fs.writeFileSync(resolve('release/prepopulatedfs.tgz'), Buffer.from(await dataDirArchive.arrayBuffer()))
14+
console.info('Success writing file to disk')
15+
} catch (e) {
16+
console.error(e)
17+
}
18+
await pglite.close()
19+
console.info('PGlite closed')
20+

0 commit comments

Comments
 (0)