Merge pull request #34 from agnosticeng/chore/sources
feat: use chdb to handle sources
This commit is contained in:
Didier Franc
116
src-tauri/Cargo.lock
generated
116
src-tauri/Cargo.lock
generated
@@ -31,7 +31,6 @@ dependencies = [
|
||||
"tauri",
|
||||
"tauri-build",
|
||||
"tauri-plugin-context-menu",
|
||||
"tauri-plugin-store",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
@@ -1810,17 +1809,6 @@ dependencies = [
|
||||
"objc_exception",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc-foundation"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1add1b659e36c9607c7aab864a76c7a4c2760cd0cd2e120f3fb8b952c7e22bf9"
|
||||
dependencies = [
|
||||
"block",
|
||||
"objc",
|
||||
"objc_id",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc_exception"
|
||||
version = "0.1.2"
|
||||
@@ -2386,30 +2374,6 @@ version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
|
||||
|
||||
[[package]]
|
||||
name = "rfd"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0149778bd99b6959285b0933288206090c50e2327f47a9c463bfdbf45c8823ea"
|
||||
dependencies = [
|
||||
"block",
|
||||
"dispatch",
|
||||
"glib-sys",
|
||||
"gobject-sys",
|
||||
"gtk-sys",
|
||||
"js-sys",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"objc",
|
||||
"objc-foundation",
|
||||
"objc_id",
|
||||
"raw-window-handle",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
"windows 0.37.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.24"
|
||||
@@ -2902,7 +2866,6 @@ dependencies = [
|
||||
"rand 0.8.5",
|
||||
"raw-window-handle",
|
||||
"regex",
|
||||
"rfd",
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -3004,18 +2967,6 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tauri-plugin-store"
|
||||
version = "0.0.0"
|
||||
source = "git+https://github.com/tauri-apps/plugins-workspace?branch=v1#8244294335a7567fe30b2aa577b764991262b73f"
|
||||
dependencies = [
|
||||
"log",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tauri",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tauri-runtime"
|
||||
version = "0.14.5"
|
||||
@@ -3506,18 +3457,6 @@ dependencies = [
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-futures"
|
||||
version = "0.4.43"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.93"
|
||||
@@ -3547,16 +3486,6 @@ version = "0.2.93"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.70"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webkit2gtk"
|
||||
version = "0.18.2"
|
||||
@@ -3670,7 +3599,7 @@ version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
|
||||
dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3679,19 +3608,6 @@ version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.37.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57b543186b344cc61c85b5aab0d2e3adf4e0f99bc076eff9aa5927bcc0b8a647"
|
||||
dependencies = [
|
||||
"windows_aarch64_msvc 0.37.0",
|
||||
"windows_i686_gnu 0.37.0",
|
||||
"windows_i686_msvc 0.37.0",
|
||||
"windows_x86_64_gnu 0.37.0",
|
||||
"windows_x86_64_msvc 0.37.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.39.0"
|
||||
@@ -3856,12 +3772,6 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.37.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2623277cb2d1c216ba3b578c0f3cf9cdebeddb6e66b1b218bb33596ea7769c3a"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.39.0"
|
||||
@@ -3886,12 +3796,6 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.37.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d3925fd0b0b804730d44d4b6278c50f9699703ec49bcd628020f46f4ba07d9e1"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.39.0"
|
||||
@@ -3922,12 +3826,6 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.37.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce907ac74fe331b524c1298683efbf598bb031bc84d5e274db2083696d07c57c"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.39.0"
|
||||
@@ -3952,12 +3850,6 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.37.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2babfba0828f2e6b32457d5341427dcbb577ceef556273229959ac23a10af33d"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.39.0"
|
||||
@@ -4000,12 +3892,6 @@ version = "0.52.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.37.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f4dd6dc7df2d84cf7b33822ed5b86318fb1781948e9663bacd047fc9dd52259d"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.39.0"
|
||||
|
||||
@@ -11,9 +11,8 @@ tauri-build = { version = "1", features = [] }
|
||||
bindgen = "0.70.1"
|
||||
|
||||
[dependencies]
|
||||
tauri = { version = "1", features = [ "dialog-open", "window-unmaximize", "window-start-dragging", "window-maximize", "shell-open"] }
|
||||
tauri = { version = "1", features = [ "window-unmaximize", "window-start-dragging", "window-maximize", "shell-open" ] }
|
||||
tauri-plugin-context-menu = "0.8.2"
|
||||
tauri-plugin-store = { git = "https://github.com/tauri-apps/plugins-workspace", branch = "v1" }
|
||||
thiserror = "1"
|
||||
|
||||
[features]
|
||||
|
||||
@@ -20,7 +20,6 @@ struct AppState {
|
||||
|
||||
fn main() {
|
||||
tauri::Builder::default()
|
||||
.plugin(tauri_plugin_store::Builder::default().build())
|
||||
.plugin(tauri_plugin_context_menu::init())
|
||||
.setup(|app| {
|
||||
let working_dir = app.path_resolver().app_local_data_dir().unwrap();
|
||||
|
||||
@@ -23,10 +23,6 @@
|
||||
"maximize": true,
|
||||
"unmaximize": true,
|
||||
"startDragging": true
|
||||
},
|
||||
"dialog": {
|
||||
"all": false,
|
||||
"open": true
|
||||
}
|
||||
},
|
||||
"windows": [
|
||||
|
||||
12
src/lib/ch-engine/index.ts
Normal file
12
src/lib/ch-engine/index.ts
Normal file
@@ -0,0 +1,12 @@
|
||||
import { exec } from './query';
|
||||
|
||||
export async function init() {
|
||||
await exec(`
|
||||
CREATE DATABASE IF NOT EXISTS agx;
|
||||
USE agx;
|
||||
`);
|
||||
}
|
||||
|
||||
export { Sources } from './sources.svelte';
|
||||
export type * from './types';
|
||||
export { exec };
|
||||
@@ -1,17 +1,13 @@
|
||||
import { invoke } from '@tauri-apps/api/tauri';
|
||||
import type { CHResponse } from './types';
|
||||
|
||||
export async function exec(query: string) {
|
||||
try {
|
||||
const r: string = await invoke('query', {
|
||||
query
|
||||
});
|
||||
const r: string = await invoke('query', { query });
|
||||
if (!r) return;
|
||||
|
||||
return JSON.parse(r) as CHResponse;
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
export type CHResponse = {
|
||||
meta: Array<{ name: string; type: string }>;
|
||||
data: Array<{ [key: string]: any }>;
|
||||
};
|
||||
32
src/lib/ch-engine/sources.svelte.ts
Normal file
32
src/lib/ch-engine/sources.svelte.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
import { exec } from './query';
|
||||
import type { Source } from './types';
|
||||
|
||||
const LIST_SOURCES_QUERY = `select
|
||||
t.name as name,
|
||||
t.engine as engine,
|
||||
groupArray(map(
|
||||
'name', c.name,
|
||||
'type', c.type
|
||||
)) as columns
|
||||
from system.tables as t
|
||||
inner join system.columns as c on t.name = c.table
|
||||
where database = 'agx'
|
||||
group by t.name, t.engine
|
||||
`;
|
||||
|
||||
export class Sources {
|
||||
#tables = $state.raw<Source[]>([]);
|
||||
|
||||
constructor() {
|
||||
this.fetch();
|
||||
}
|
||||
|
||||
public get tables() {
|
||||
return this.#tables;
|
||||
}
|
||||
|
||||
async fetch() {
|
||||
const response = await exec(LIST_SOURCES_QUERY);
|
||||
if (response) this.#tables = response.data as Source[];
|
||||
}
|
||||
}
|
||||
15
src/lib/ch-engine/types.d.ts
vendored
Normal file
15
src/lib/ch-engine/types.d.ts
vendored
Normal file
@@ -0,0 +1,15 @@
|
||||
export type CHResponse = {
|
||||
meta: Array<ColumnDescriptor>;
|
||||
data: Array<{ [key: string]: any }>;
|
||||
};
|
||||
|
||||
export interface ColumnDescriptor {
|
||||
name: string;
|
||||
type: string;
|
||||
}
|
||||
|
||||
export interface Source {
|
||||
name: string;
|
||||
engine: string;
|
||||
columns: ColumnDescriptor[];
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
<script lang="ts">
|
||||
import type { CHResponse } from '$lib/ch-engine';
|
||||
import { applyType, formatValue, isSupportedType, LineChart } from '$lib/components/charts/Line';
|
||||
import type { CHResponse } from '$lib/query';
|
||||
import { BarChart } from './charts/Bar';
|
||||
|
||||
interface Props {
|
||||
|
||||
@@ -1,224 +0,0 @@
|
||||
<script lang="ts" module>
|
||||
export type FormValues = {
|
||||
name: string;
|
||||
slug: string;
|
||||
path: string;
|
||||
};
|
||||
</script>
|
||||
|
||||
<script lang="ts">
|
||||
import Modal from '$lib/components/Modal.svelte';
|
||||
import { slugify } from '$lib/slugify';
|
||||
import type { MaybePromise } from '$lib/types';
|
||||
import { dialog } from '@tauri-apps/api';
|
||||
import { type UnlistenFn } from '@tauri-apps/api/event';
|
||||
import { appWindow } from '@tauri-apps/api/window';
|
||||
import { onDestroy, tick } from 'svelte';
|
||||
|
||||
interface Props {
|
||||
onCreate?: (values: FormValues) => MaybePromise<unknown>;
|
||||
}
|
||||
|
||||
let { onCreate }: Props = $props();
|
||||
|
||||
let open = $state(false);
|
||||
let error_message = $state('');
|
||||
|
||||
export function show() {
|
||||
open = true;
|
||||
}
|
||||
|
||||
let modal = $state<ReturnType<typeof Modal>>();
|
||||
|
||||
async function handleSubmit(e: SubmitEvent & { currentTarget: HTMLFormElement }) {
|
||||
e.preventDefault();
|
||||
error_message = '';
|
||||
const form_data = new FormData(e.currentTarget);
|
||||
const values = Object.fromEntries(form_data) as FormValues;
|
||||
|
||||
try {
|
||||
await onCreate?.(values);
|
||||
modal?.close();
|
||||
} catch (e) {
|
||||
if (e instanceof TypeError) {
|
||||
error_message = e.message;
|
||||
}
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
let name_value = $state('');
|
||||
let path_value = $state('');
|
||||
|
||||
$effect(() => {
|
||||
open;
|
||||
|
||||
name_value = '';
|
||||
path_value = '';
|
||||
error_message = '';
|
||||
});
|
||||
|
||||
async function open_file() {
|
||||
const path = await dialog.open({
|
||||
title: 'Open Source',
|
||||
filters: [{ name: 'source files', extensions: ['csv', 'parquet'] }],
|
||||
multiple: false
|
||||
});
|
||||
|
||||
if (typeof path === 'string') {
|
||||
name_value = path.split('/').pop() ?? '';
|
||||
path_value = 'file://' + path;
|
||||
}
|
||||
}
|
||||
|
||||
let unlisten = $state<UnlistenFn>();
|
||||
|
||||
appWindow
|
||||
.onFileDropEvent(async (event) => {
|
||||
if (event.payload.type !== 'drop') return;
|
||||
if (event.payload.paths.length !== 1) return;
|
||||
const [path] = event.payload.paths;
|
||||
const ext = path.split('.').pop();
|
||||
if (!ext || !['csv', 'parquet'].includes(ext.toLowerCase())) return;
|
||||
if (!open) {
|
||||
open = true;
|
||||
await tick();
|
||||
}
|
||||
|
||||
name_value = path.split('/').pop() ?? '';
|
||||
path_value = 'file://' + path;
|
||||
})
|
||||
.then((unlistenFn) => (unlisten = unlistenFn));
|
||||
|
||||
onDestroy(() => unlisten?.());
|
||||
</script>
|
||||
|
||||
{#if open}
|
||||
<Modal onclose={() => (open = false)} bind:this={modal}>
|
||||
<form onsubmit={handleSubmit}>
|
||||
<h2>Add Source</h2>
|
||||
<label>
|
||||
<span>Name:</span>
|
||||
<input
|
||||
type="text"
|
||||
placeholder="Ethereum events"
|
||||
spellcheck="false"
|
||||
autocomplete="off"
|
||||
name="name"
|
||||
bind:value={name_value}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
<span>Alias:</span>
|
||||
<input
|
||||
type="text"
|
||||
placeholder="ethereum_events"
|
||||
name="slug"
|
||||
value={slugify(name_value)}
|
||||
readonly
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
<span>File path:</span>
|
||||
<div>
|
||||
<input
|
||||
type="text"
|
||||
placeholder="s3://data.agnostic.dev/ethereum-mainnet-pq/logs/*.parquet"
|
||||
spellcheck="false"
|
||||
autocomplete="off"
|
||||
name="path"
|
||||
bind:value={path_value}
|
||||
required
|
||||
/>
|
||||
<button type="button" onclick={open_file}>Choose file</button>
|
||||
</div>
|
||||
</label>
|
||||
|
||||
<div class="Error">
|
||||
{#if error_message}
|
||||
{error_message}
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<div class="Actions">
|
||||
<button type="button" onclick={() => modal!.close()}>Cancel</button>
|
||||
<button type="submit">Add source</button>
|
||||
</div>
|
||||
</form>
|
||||
</Modal>
|
||||
{/if}
|
||||
|
||||
<style>
|
||||
h2 {
|
||||
margin-top: 0;
|
||||
}
|
||||
|
||||
form {
|
||||
padding: 2rem;
|
||||
}
|
||||
|
||||
form label {
|
||||
display: block;
|
||||
|
||||
& ~ label {
|
||||
margin-top: 12px;
|
||||
}
|
||||
|
||||
& > span {
|
||||
display: block;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
}
|
||||
|
||||
input[type='text'] {
|
||||
width: 100%;
|
||||
background-color: hsl(0deg 0% 3%);
|
||||
border: 1px solid hsl(0deg 0% 3%);
|
||||
border-radius: 3px;
|
||||
padding: 5px 10px;
|
||||
|
||||
outline: none;
|
||||
caret-color: currentcolor;
|
||||
|
||||
&:not(:read-only):is(:focus-within, :hover) {
|
||||
border-color: hsl(0deg 0% 34%);
|
||||
}
|
||||
}
|
||||
|
||||
label > div {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
button {
|
||||
appearance: none;
|
||||
outline: none;
|
||||
border: none;
|
||||
text-wrap: nowrap;
|
||||
background-color: hsl(0deg 0% 33%);
|
||||
padding: 2px 8px;
|
||||
border-radius: 3px;
|
||||
cursor: pointer;
|
||||
|
||||
&:is(:active) {
|
||||
background-color: hsl(0deg 0% 52%);
|
||||
}
|
||||
}
|
||||
|
||||
div.Error {
|
||||
font-size: 0.9rem;
|
||||
text-align: center;
|
||||
color: hsl(0 100% 70%);
|
||||
height: 0.9rem;
|
||||
padding: 6px 0;
|
||||
}
|
||||
|
||||
div.Actions {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
gap: 12px;
|
||||
height: 28px;
|
||||
margin-top: 12px;
|
||||
}
|
||||
</style>
|
||||
@@ -2,74 +2,45 @@
|
||||
import SearchBar from '$lib/components/SearchBar.svelte';
|
||||
import { get_app_context } from '$lib/context';
|
||||
import Database from '$lib/icons/Database.svelte';
|
||||
import Plus from '$lib/icons/Plus.svelte';
|
||||
import Table from '$lib/icons/Table.svelte';
|
||||
import { listen } from '@tauri-apps/api/event';
|
||||
import { showMenu } from 'tauri-plugin-context-menu';
|
||||
import AddDataset from './AddDataset.svelte';
|
||||
import {
|
||||
DATASOURCE_TYPE_COLOR_MAP,
|
||||
DATASOURCE_TYPE_SHORT_NAME_MAP,
|
||||
DEFAULT_SOURCE,
|
||||
filter,
|
||||
remove_nullable
|
||||
remove_nullable,
|
||||
SOURCE_TYPE_COLOR_MAP,
|
||||
SOURCE_TYPE_SHORT_NAME_MAP
|
||||
} from './utils';
|
||||
|
||||
const { datasets } = get_app_context();
|
||||
const { sources } = get_app_context();
|
||||
|
||||
let loading = $state(false);
|
||||
|
||||
let search = $state<string>('');
|
||||
const filtered = $derived(filter(datasets.sources, search));
|
||||
|
||||
let add_dataset_modal: ReturnType<typeof AddDataset>;
|
||||
const filtered = $derived(filter(sources.tables, search));
|
||||
</script>
|
||||
|
||||
<SearchBar bind:value={search} />
|
||||
<div>
|
||||
<button
|
||||
disabled={loading}
|
||||
onclick={() => {
|
||||
if (loading) return;
|
||||
loading = true;
|
||||
sources.fetch().finally(() => (loading = false));
|
||||
}}>Refresh</button
|
||||
>
|
||||
</div>
|
||||
<article>
|
||||
{#each filtered as source, i (source.slug)}
|
||||
{#each filtered as source, i (source.name)}
|
||||
<details open={i === 0}>
|
||||
<summary
|
||||
oncontextmenu={async (e) => {
|
||||
e.preventDefault();
|
||||
const element = e.currentTarget;
|
||||
element.classList.add('Selected');
|
||||
await showMenu({
|
||||
theme: 'dark',
|
||||
items: [
|
||||
{
|
||||
label: 'Reload',
|
||||
event: () => datasets.refresh(source)
|
||||
},
|
||||
{
|
||||
label: 'Copy path',
|
||||
event: () => navigator.clipboard.writeText(source.path)
|
||||
},
|
||||
{
|
||||
label: 'Copy alias',
|
||||
event: () => navigator.clipboard.writeText(source.slug)
|
||||
},
|
||||
{ is_separator: true },
|
||||
{
|
||||
label: 'Remove',
|
||||
event: () => datasets.remove(source),
|
||||
disabled: source.slug === DEFAULT_SOURCE.slug
|
||||
}
|
||||
]
|
||||
});
|
||||
|
||||
const unlistenMenuClose = await listen('menu-did-close', () => {
|
||||
element.classList.remove('Selected');
|
||||
unlistenMenuClose();
|
||||
});
|
||||
}}
|
||||
>
|
||||
{#if source.type === 'MergeTree'}
|
||||
<summary>
|
||||
{#if source.engine === 'MergeTree'}
|
||||
<Database size="15" />
|
||||
{:else}
|
||||
<Table size="15" />
|
||||
{/if}
|
||||
<h3>{source.name}</h3>
|
||||
<span class="Tag" style:background-color={DATASOURCE_TYPE_COLOR_MAP[source.type]}>
|
||||
{DATASOURCE_TYPE_SHORT_NAME_MAP[source.type]}
|
||||
<span class="Tag" style:background-color={SOURCE_TYPE_COLOR_MAP[source.engine]}>
|
||||
{SOURCE_TYPE_SHORT_NAME_MAP[source.engine]}
|
||||
</span>
|
||||
</summary>
|
||||
<ul>
|
||||
@@ -82,19 +53,8 @@
|
||||
</ul>
|
||||
</details>
|
||||
{/each}
|
||||
<div class="Actions">
|
||||
<button onclick={() => add_dataset_modal.show()}><Plus size="12" /></button>
|
||||
</div>
|
||||
</article>
|
||||
|
||||
<AddDataset
|
||||
bind:this={add_dataset_modal}
|
||||
onCreate={async (values) => {
|
||||
const index = datasets.sources.findIndex((s) => s.slug === values.slug);
|
||||
if (index === -1) datasets.add({ name: values.name, slug: values.slug, path_url: values.path });
|
||||
}}
|
||||
/>
|
||||
|
||||
<style>
|
||||
article {
|
||||
flex: 1;
|
||||
@@ -124,8 +84,17 @@
|
||||
display: none;
|
||||
}
|
||||
|
||||
&:global(.Selected) {
|
||||
background-color: hsl(210deg 100% 52%);
|
||||
& > :global(svg),
|
||||
& > span {
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
& > h3 {
|
||||
flex-shrink: 1;
|
||||
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -186,27 +155,4 @@
|
||||
padding-bottom: 0;
|
||||
}
|
||||
}
|
||||
|
||||
div {
|
||||
margin-top: 35px;
|
||||
|
||||
& > button {
|
||||
/* Reset */
|
||||
appearance: none;
|
||||
outline: none;
|
||||
border: none;
|
||||
|
||||
background-color: hsl(0deg 0% 33%);
|
||||
padding: 2px 8px;
|
||||
border-radius: 3px;
|
||||
cursor: pointer;
|
||||
|
||||
display: flex;
|
||||
align-items: center;
|
||||
|
||||
&:is(:active) {
|
||||
background-color: hsl(0deg 0% 52%);
|
||||
}
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -1,62 +1,28 @@
|
||||
import { exec, type CHResponse } from '$lib/query';
|
||||
import type { ColumnDescriptor, Dataset } from '$lib/types';
|
||||
import { type Source } from '$lib/ch-engine';
|
||||
|
||||
export function filter(sources: Dataset[], search: string) {
|
||||
export function filter(sources: Source[], search: string) {
|
||||
if (!search) return sources;
|
||||
const search_ = search.toLowerCase();
|
||||
|
||||
return sources.filter(
|
||||
(s) =>
|
||||
s.name.toLowerCase().includes(search_) ||
|
||||
s.slug.includes(search_) ||
|
||||
s.columns?.some((c) => c.name.toLowerCase().includes(search_))
|
||||
);
|
||||
}
|
||||
|
||||
export const DATASOURCE_TYPE_COLOR_MAP: Record<Dataset['type'], string> = {
|
||||
CSV: 'hsl(58deg 37% 28%)',
|
||||
Parquet: 'hsl(20deg 37% 28%)',
|
||||
export const SOURCE_TYPE_COLOR_MAP: Record<string, string> = {
|
||||
FILE: 'hsl(58deg 37% 28%)',
|
||||
S3: 'hsl(20deg 37% 28%)',
|
||||
MergeTree: 'hsl(199deg 37% 28%)'
|
||||
};
|
||||
|
||||
export const DATASOURCE_TYPE_SHORT_NAME_MAP: Record<Dataset['type'], string> = {
|
||||
CSV: 'CSV',
|
||||
Parquet: 'PQT',
|
||||
export const SOURCE_TYPE_SHORT_NAME_MAP: Record<string, string> = {
|
||||
FILE: 'FILE',
|
||||
S3: 'S3',
|
||||
MergeTree: 'MT'
|
||||
};
|
||||
|
||||
export function remove_nullable(type: string) {
|
||||
return type.replace(/Nullable\((.*)\)/, '$1');
|
||||
}
|
||||
|
||||
export function describe_to_column_descriptors(response: CHResponse): ColumnDescriptor[] {
|
||||
return response.data.map((d) => {
|
||||
return {
|
||||
name: d.name as string,
|
||||
type: d.type as string
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export async function getDefaultSource() {
|
||||
const defaults: Dataset = {
|
||||
name: 'Agnostic Logs',
|
||||
slug: 'agnostic_logs',
|
||||
path: "s3('https://data.agnostic.dev/ethereum-mainnet-pq/logs/*.parquet', 'Parquet')",
|
||||
type: 'Parquet',
|
||||
last_refresh: Date.now()
|
||||
};
|
||||
|
||||
const response = await exec(`DESCRIBE TABLE ${defaults.path}`);
|
||||
if (!response) return;
|
||||
|
||||
defaults.columns = describe_to_column_descriptors(response);
|
||||
|
||||
return defaults;
|
||||
}
|
||||
|
||||
export const DEFAULT_SOURCE = {
|
||||
name: 'Agnostic Logs',
|
||||
slug: 'agnostic_logs',
|
||||
path_url: 's3://data.agnostic.dev/ethereum-mainnet-pq/logs/*.parquet'
|
||||
};
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
<script lang="ts">
|
||||
import type { ColumnDescriptor } from '$lib/ch-engine';
|
||||
import { sql } from '@codemirror/lang-sql';
|
||||
import { Compartment, EditorState } from '@codemirror/state';
|
||||
import { EditorView, keymap, placeholder } from '@codemirror/view';
|
||||
@@ -6,7 +7,6 @@
|
||||
import './codemirror.css';
|
||||
import { default_extensions, default_keymaps } from './extensions';
|
||||
import { ProxyDialect } from './SQLDialect';
|
||||
import type { ColumnDescriptor } from '$lib/types';
|
||||
import { schema_to_completions } from './utils';
|
||||
|
||||
type Props = {
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
export { default as Editor } from './Editor.svelte';
|
||||
export { datasets_to_schema } from './utils';
|
||||
export { sources_to_schema } from './utils';
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import type { ColumnDescriptor, Dataset } from '$lib/types';
|
||||
import type { ColumnDescriptor, Source } from '$lib/ch-engine';
|
||||
import type { Completion } from '@codemirror/autocomplete';
|
||||
|
||||
export function datasets_to_schema(datasets: Dataset[]): {
|
||||
export function sources_to_schema(sources: Source[]): {
|
||||
[table_name: string]: ColumnDescriptor[];
|
||||
} {
|
||||
return datasets.reduce((acc, k) => {
|
||||
if (k.columns?.length) return { ...acc, [k.slug]: k.columns };
|
||||
return sources.reduce((acc, k) => {
|
||||
if (k.columns.length) return { ...acc, [k.name]: k.columns };
|
||||
return acc;
|
||||
}, {});
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
<script lang="ts">
|
||||
import type { CHResponse } from '$lib/ch-engine';
|
||||
import { Table } from '$lib/components/Table';
|
||||
import type { CHResponse } from '$lib/query';
|
||||
import { untrack } from 'svelte';
|
||||
import ChartContainer from './ChartContainer.svelte';
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<script lang="ts">
|
||||
import type { CHResponse } from '$lib/query';
|
||||
import type { CHResponse } from '$lib/ch-engine';
|
||||
import { Types } from './utils';
|
||||
|
||||
interface Props {
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
import { describe_to_column_descriptors } from './components/Datasets/utils';
|
||||
import { exec } from './query';
|
||||
import type { Dataset, MaybePromise } from './types';
|
||||
|
||||
export interface RawSource {
|
||||
/** Name of the Dataset */
|
||||
name: string;
|
||||
/** slug that will be used to identify the dataset in the queries */
|
||||
slug: string;
|
||||
/** URL like, example: s3://data.agnostic.dev/ethereum-mainnet-pq/logs/*.parquet or file:///Users/johndoe/data/logs.csv */
|
||||
path_url: string;
|
||||
}
|
||||
|
||||
export interface DatasetsConfig {
|
||||
onupdate?: (dataset: Dataset) => MaybePromise<void>;
|
||||
onreset?: (datasets: Dataset[]) => MaybePromise<void>;
|
||||
}
|
||||
|
||||
export class Datasets {
|
||||
#sources = $state.raw<Dataset[]>([]);
|
||||
|
||||
#onupdate: NonNullable<DatasetsConfig['onupdate']>;
|
||||
#onreset: NonNullable<DatasetsConfig['onreset']>;
|
||||
|
||||
constructor(sources: Dataset[], { onreset, onupdate }: DatasetsConfig = {}) {
|
||||
this.#sources = sources;
|
||||
|
||||
this.#onupdate = onupdate ?? (() => {});
|
||||
this.#onreset = onreset ?? (() => {});
|
||||
}
|
||||
|
||||
get sources() {
|
||||
return this.#sources;
|
||||
}
|
||||
|
||||
add(source: RawSource) {
|
||||
if (!validate_url(source.path_url)) throw new TypeError('Invalid path format');
|
||||
|
||||
const { path, type } = parse_path_url(source.path_url);
|
||||
|
||||
const new_source = {
|
||||
name: source.name,
|
||||
slug: source.slug,
|
||||
path: path,
|
||||
last_refresh: -1,
|
||||
type: type
|
||||
};
|
||||
this.#sources = this.#sources.concat(new_source);
|
||||
|
||||
this.refresh(new_source);
|
||||
}
|
||||
|
||||
update(source: Dataset) {
|
||||
this.#sources = this.#sources.map((old) => {
|
||||
if (old.slug === source.slug) {
|
||||
return source;
|
||||
}
|
||||
|
||||
return old;
|
||||
});
|
||||
|
||||
this.#onupdate(source);
|
||||
}
|
||||
|
||||
remove(source: Dataset) {
|
||||
const index = this.#sources.findIndex((s) => s.slug === source.slug);
|
||||
if (index === -1)
|
||||
throw new Error('Tried to remove a source that is not registered in the datasets');
|
||||
|
||||
this.#sources = this.#sources.filter((s) => s.slug !== source.slug);
|
||||
|
||||
this.#onreset(this.#sources);
|
||||
}
|
||||
|
||||
async refresh(source: Dataset) {
|
||||
const index = this.#sources.findIndex((s) => s.slug === source.slug);
|
||||
if (index === -1)
|
||||
throw new Error('Tried to refresh a source that is not registered in the datasets');
|
||||
|
||||
const response = await exec(`DESCRIBE ${source.path}`);
|
||||
if (!response) throw new Error(`Cannot update '${source.name}' source`);
|
||||
const columns = describe_to_column_descriptors(response);
|
||||
|
||||
this.update({ ...source, columns, last_refresh: Date.now() });
|
||||
}
|
||||
}
|
||||
|
||||
const S3_REGEXP =
|
||||
/s3:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=*]+)/;
|
||||
|
||||
const FILE_REGEXP = /file:\/\/([-a-zA-Z0-9()@:%_\+.~#?&// ;=*]+)/;
|
||||
|
||||
function validate_url(_path: string) {
|
||||
return S3_REGEXP.test(_path) || FILE_REGEXP.test(_path);
|
||||
}
|
||||
|
||||
function parse_path_url(path: string) {
|
||||
const protocol_func = new URL(path).protocol.replace(':', '').toLowerCase();
|
||||
const url = path.replace(/s3/, 'https').replace(/file:\/\//, '');
|
||||
const extension = url.split('.').pop();
|
||||
if (!extension) throw new TypeError('Invalid path format: extension not specified');
|
||||
const file_type = extract_file_type(extension);
|
||||
if (!file_type)
|
||||
throw new TypeError(`Invalid path format: unsupported extension ${file_type ?? ''}`.trim());
|
||||
|
||||
return { path: `${protocol_func}('${url}', '${file_type}')`, type: file_type };
|
||||
}
|
||||
|
||||
function extract_file_type(extension: string): 'CSV' | 'Parquet' | undefined {
|
||||
switch (extension.toLowerCase()) {
|
||||
case 'csv':
|
||||
return 'CSV';
|
||||
case 'parquet':
|
||||
return 'Parquet';
|
||||
}
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
import { Store } from 'tauri-plugin-store-api';
|
||||
import type { Dataset } from './types';
|
||||
|
||||
const path = 'datasets.json';
|
||||
|
||||
const sources_key = 'sources';
|
||||
|
||||
const store = new Store(path);
|
||||
|
||||
export async function get_sources_from_store(): Promise<Dataset[]> {
|
||||
return (await store.get<Dataset[]>(sources_key)) ?? [];
|
||||
}
|
||||
|
||||
export async function set_sources_in_store(sources: Dataset[]) {
|
||||
await store.set(sources_key, sources);
|
||||
}
|
||||
21
src/lib/types.d.ts
vendored
21
src/lib/types.d.ts
vendored
@@ -1,24 +1,7 @@
|
||||
import type { Datasets } from './sources.svelte';
|
||||
|
||||
export interface ColumnDescriptor {
|
||||
name: string;
|
||||
type: string;
|
||||
}
|
||||
|
||||
export interface Dataset {
|
||||
name: string;
|
||||
/** Must be unique */
|
||||
slug: string;
|
||||
path: string;
|
||||
type: 'CSV' | 'Parquet' | 'MergeTree';
|
||||
/** Describe result */
|
||||
columns?: ColumnDescriptor[];
|
||||
/** Timestamp */
|
||||
last_refresh: number;
|
||||
}
|
||||
import type { Sources } from './ch-engine';
|
||||
|
||||
type MaybePromise<T> = T | Promise<T>;
|
||||
|
||||
export type AppContext = {
|
||||
datasets: Datasets;
|
||||
sources: Sources;
|
||||
};
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
import type { Dataset } from '$lib/types';
|
||||
|
||||
export function applySlugs(query: string, datasets: Dataset[]) {
|
||||
let q = query;
|
||||
|
||||
for (const dataset of datasets) {
|
||||
q = q.replace(new RegExp(`(from|FROM)[ \n\t]+(${dataset.slug})`, 'g'), (match) =>
|
||||
match.replace(dataset.slug, `${dataset.path} ${dataset.slug}`)
|
||||
);
|
||||
|
||||
q = q.replace(
|
||||
new RegExp(`(describe|DESCRIBE)([ \n\t]+(table|TABLE))?[ \n\t]+(${dataset.slug})`, 'g'),
|
||||
(match) => match.replace(dataset.slug, `${dataset.path}`)
|
||||
);
|
||||
}
|
||||
|
||||
return q;
|
||||
}
|
||||
@@ -1,15 +1,11 @@
|
||||
<script lang="ts">
|
||||
import { DEFAULT_SOURCE } from '$lib/components/Datasets/utils';
|
||||
import { Editor } from '$lib/components/Editor';
|
||||
import { exec, Sources, type CHResponse } from '$lib/ch-engine';
|
||||
import { Editor, sources_to_schema } from '$lib/components/Editor';
|
||||
import Result from '$lib/components/Result.svelte';
|
||||
import SideBar from '$lib/components/SideBar.svelte';
|
||||
import { SplitPane } from '$lib/components/SplitPane';
|
||||
import WindowTitleBar from '$lib/components/WindowTitleBar.svelte';
|
||||
import { set_app_context } from '$lib/context';
|
||||
import { exec, type CHResponse } from '$lib/query';
|
||||
import { Datasets } from '$lib/sources.svelte';
|
||||
import { set_sources_in_store } from '$lib/store';
|
||||
import { applySlugs } from '$lib/utils/datasets';
|
||||
import type { PageData } from './$types';
|
||||
|
||||
let response = $state.raw<CHResponse>();
|
||||
@@ -22,25 +18,12 @@
|
||||
async function handleExec() {
|
||||
if (loading) return;
|
||||
loading = true;
|
||||
response = await exec(applySlugs(query, datasets.sources)).finally(() => (loading = false));
|
||||
response = await exec(query).finally(() => (loading = false));
|
||||
}
|
||||
|
||||
const datasets = new Datasets(data.sources, {
|
||||
onreset(datasets) {
|
||||
set_sources_in_store(datasets);
|
||||
},
|
||||
onupdate(_dataset) {
|
||||
set_sources_in_store(datasets.sources);
|
||||
}
|
||||
});
|
||||
const sources = new Sources();
|
||||
|
||||
set_app_context({ datasets });
|
||||
|
||||
$effect.pre(() => {
|
||||
if (!datasets.sources.length) {
|
||||
datasets.add(DEFAULT_SOURCE);
|
||||
}
|
||||
});
|
||||
set_app_context({ sources });
|
||||
</script>
|
||||
|
||||
<WindowTitleBar>
|
||||
@@ -57,7 +40,11 @@
|
||||
{#snippet b()}
|
||||
<SplitPane orientation="vertical" min="20%" max="80%" --color="hsl(0deg 0% 12%)">
|
||||
{#snippet a()}
|
||||
<Editor bind:value={query} onExec={handleExec} />
|
||||
<Editor
|
||||
bind:value={query}
|
||||
onExec={handleExec}
|
||||
schema={sources_to_schema(sources.tables)}
|
||||
/>
|
||||
{/snippet}
|
||||
{#snippet b()}
|
||||
<Result {response} />
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { get_sources_from_store } from '$lib/store';
|
||||
import { init } from '$lib/ch-engine';
|
||||
import type { PageLoad } from './$types';
|
||||
|
||||
export const load = (async () => {
|
||||
return { sources: await get_sources_from_store() };
|
||||
await init();
|
||||
|
||||
return {};
|
||||
}) satisfies PageLoad;
|
||||
|
||||
Reference in New Issue
Block a user