dave/clickhouse-agx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore(datasets): update dataset list

Browse Source
This commit is contained in:
Yann Amsellem
2024-11-27 14:57:28 +01:00
parent 2fdf20ada2
commit d136535505
14 changed files with 440 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src-tauri/src/commands/mod.rs
View File

@@ -7,7 +7,7 @@ use crate::AppState;
use String;
#[tauri::command]
pub async fn query(query: String, state: State<'_, Mutex<AppState>>) -> Result<String, ()> {
pub async fn query(query: String, state: State<'_, Mutex<AppState>>) -> Result<String, String> {
let state = state.lock().unwrap();
let args = vec![
@@ -21,9 +21,9 @@ pub async fn query(query: String, state: State<'_, Mutex<AppState>>) -> Result<S
return match lib::execute(&query, Some(&args)) {
Ok(Some(query_result)) => query_result
.data_utf8()
.map_err(|_| ())
.map_err(|_| "Bad encoding".to_string())
.map(|data| data.to_string()),
Ok(None) => Ok(String::from("No result")),
Err(_e) => Err(()),
Err(_e) => Err(_e.to_string()),
};
}

156
src/lib/components/Datasets/Datasets.svelte Normal file
View File

@@ -0,0 +1,156 @@
<script lang="ts">
import Database from '$lib/icons/Database.svelte';
import Plus from '$lib/icons/Plus.svelte';
import Table from '$lib/icons/Table.svelte';
import type { Dataset } from '$lib/types';
import SearchBar from '../SearchBar.svelte';
import {
DATASOURCE_TYPE_COLOR_MAP,
DATASOURCE_TYPE_SHORT_NAME_MAP,
filter,
remove_nullable
} from './utils';
interface Props {
sources: Dataset[];
}
let { sources }: Props = $props();
let search = $state<string>('');
const filtered = $derived(filter(sources, search));
</script>
<SearchBar bind:value={search} />
<article>
{#each filtered as source, i (source.slug)}
<details open={i === 0}>
<summary>
{#if source.type === 'MergeTree'}
<Database size="15" />
{:else}
<Table size="15" />
{/if}
<h3>{source.name}</h3>
<span class="Tag" style:background-color={DATASOURCE_TYPE_COLOR_MAP[source.type]}>
{DATASOURCE_TYPE_SHORT_NAME_MAP[source.type]}
</span>
</summary>
<ul>
{#each source.columns ?? [] as column}
<li>
<span>{column.name}</span>
<span>{remove_nullable(column.type)}</span>
</li>
{/each}
</ul>
</details>
{/each}
<div class="Actions">
<button><Plus size="12" /></button>
</div>
</article>
<style>
article {
flex: 1;
overflow-y: auto;
}
details {
width: 100%;
details ~ & {
margin-top: 12px;
}
}
summary {
cursor: pointer;
user-select: none;
-webkit-user-select: none;
display: flex;
align-items: center;
gap: 5px;
&::-webkit-details-marker {
display: none;
}
}
h3 {
margin: 0;
}
.Tag {
display: block;
font-size: 8px;
font-weight: 500;
padding: 3px;
border-radius: 3px;
min-width: 16px;
}
ul {
/* Reset */
list-style: none;
margin: 0;
padding: 0;
/* Stack */
display: flex;
flex-direction: column;
gap: 5px;
/* Custom style */
padding: 12px 0;
padding-left: 5px;
& > li {
display: flex;
align-items: center;
& > span:first-of-type {
flex-grow: 1;
}
& > span:last-of-type {
flex-shrink: 0;
font-size: 8px;
font-weight: 500;
padding: 2px;
border-radius: 3px;
background-color: hsl(0deg 0% 19%);
text-align: center;
font-family: 'Fira Mono', monospace;
}
}
&:last-of-type {
padding-bottom: 0;
}
}
div {
margin-top: 35px;
& > button {
/* Reset */
appearance: none;
outline: none;
border: none;
background-color: hsl(0deg 0% 33%);
padding: 2px 8px;
border-radius: 3px;
cursor: pointer;
display: flex;
align-items: center;
&:is(:active) {
background-color: hsl(0deg 0% 52%);
}
}
}
</style>

58
src/lib/components/Datasets/utils.ts Normal file
View File

@@ -0,0 +1,58 @@
import { exec, type CHResponse } from '$lib/query';
import type { ColumnDescriptor, Dataset } from '$lib/types';
export function filter(sources: Dataset[], search: string) {
if (!search) return sources;
const search_ = search.toLowerCase();
return sources.filter(
(s) =>
s.name.toLowerCase().includes(search_) ||
s.slug.includes(search_) ||
s.columns?.some((c) => c.name.toLowerCase().includes(search_))
);
}
export const DATASOURCE_TYPE_COLOR_MAP: Record<Dataset['type'], string> = {
CSV: 'hsl(58deg 37% 28%)',
Parquet: 'hsl(20deg 37% 28%)',
MergeTree: 'hsl(199deg 37% 28%)'
};
export const DATASOURCE_TYPE_SHORT_NAME_MAP: Record<Dataset['type'], string> = {
CSV: 'CSV',
Parquet: 'PQT',
MergeTree: 'MT'
};
export function remove_nullable(type: string) {
return type.replace(/Nullable\((.*)\)/, '$1');
}
export function describe_to_column_descriptors(
response: NonNullable<CHResponse>
): ColumnDescriptor[] {
return response.data.map((d) => {
return {
name: d.name as string,
type: d.type as string
};
});
}
export async function getDefaultSource() {
const defaults: Dataset = {
name: 'Agnostic Logs',
slug: 'agnostic_logs',
path: "s3('https://data.agnostic.dev/ethereum-mainnet-pq/logs/*.parquet', 'Parquet')",
type: 'Parquet',
last_refresh: Date.now()
};
const response = await exec(`DESCRIBE TABLE ${defaults.path}`);
if (!response) return;
defaults.columns = describe_to_column_descriptors(response);
return defaults;
}

53
src/lib/components/SearchBar.svelte Normal file
View File

@@ -0,0 +1,53 @@
<script lang="ts">
import type { SvelteHTMLElements } from 'svelte/elements';
import Search from '$lib/icons/Search.svelte';
type Props = Omit<
SvelteHTMLElements['input'],
'type' | 'autocapitalize' | 'autocomplete' | 'autocorrect' | 'placeholder' | 'checked'
>;
let { value = $bindable(), ...rest }: Props = $props();
</script>
<label>
<Search size="12" />
<input
type="text"
autocapitalize="off"
autocomplete="off"
autocorrect="off"
placeholder="Search..."
bind:value
{...rest}
/>
</label>
<style>
label {
width: 100%;
background-color: hsl(0deg 0% 3%);
border: 1px solid hsl(0deg 0% 3%);
border-radius: 3px;
display: flex;
align-items: center;
gap: 10px;
padding: 5px 10px;
& > input {
height: 100%;
}
&:has(> input:is(:focus-within, :hover)) {
border-color: hsl(0deg 0% 34%);
}
}
input {
background: none;
border: none;
outline: none;
caret-color: currentcolor;
}
</style>

66
src/lib/components/SideBar.svelte Normal file
View File

@@ -0,0 +1,66 @@
<script lang="ts">
import type { Dataset } from '$lib/types';
import Datasets from './Datasets/Datasets.svelte';
interface Props {
sources?: Dataset[];
}
type Navigation = 'sources' | 'queries' | 'history';
let { sources = [] }: Props = $props();
let tab = $state<Navigation>('sources');
function navigate(next_tab: Navigation) {
tab = next_tab;
}
</script>
<section>
<nav>
<button aria-current={tab === 'sources'} onclick={() => navigate('sources')}>Sources</button>
<button aria-current={tab === 'queries'} onclick={() => navigate('queries')}>Queries</button>
<button aria-current={tab === 'history'} onclick={() => navigate('history')}>History</button>
</nav>
{#if tab === 'sources'}
<Datasets {sources} />
{/if}
</section>
<style>
section {
padding: 14px 18px;
background-color: hsl(0deg 0% 9%);
display: flex;
flex-direction: column;
gap: 18px;
}
nav {
display: flex;
justify-content: center;
gap: 12px;
& > button {
font-size: 10px;
font-weight: 500;
background-color: transparent;
padding: 4px 10px;
border-radius: 3px;
cursor: pointer;
&:is(:hover, :focus-within, [aria-current='true']) {
background-color: hsl(0deg 0% 19%);
}
}
}
button {
appearance: none;
outline: none;
border: none;
background: none;
padding: 0;
display: block;
}
</style>

16
src/lib/icons/Database.svelte Normal file
View File

@@ -0,0 +1,16 @@
<script lang="ts">
import type { SvelteHTMLElements } from 'svelte/elements';
interface Props extends Omit<SvelteHTMLElements['svg'], 'width' | 'height'> {
size?: string | number | null;
}
let { size = 24, ...rest }: Props = $props();
</script>
<svg width={size} height={size} viewBox="0 0 256 256" {...rest}>
<path
fill="currentColor"
d="M128,24C74.17,24,32,48.6,32,80v96c0,31.4,42.17,56,96,56s96-24.6,96-56V80C224,48.6,181.83,24,128,24Zm80,104c0,9.62-7.88,19.43-21.61,26.92C170.93,163.35,150.19,168,128,168s-42.93-4.65-58.39-13.08C55.88,147.43,48,137.62,48,128V111.36c17.06,15,46.23,24.64,80,24.64s62.94-9.68,80-24.64ZM69.61,53.08C85.07,44.65,105.81,40,128,40s42.93,4.65,58.39,13.08C200.12,60.57,208,70.38,208,80s-7.88,19.43-21.61,26.92C170.93,115.35,150.19,120,128,120s-42.93-4.65-58.39-13.08C55.88,99.43,48,89.62,48,80S55.88,60.57,69.61,53.08ZM186.39,202.92C170.93,211.35,150.19,216,128,216s-42.93-4.65-58.39-13.08C55.88,195.43,48,185.62,48,176V159.36c17.06,15,46.23,24.64,80,24.64s62.94-9.68,80-24.64V176C208,185.62,200.12,195.43,186.39,202.92Z"
/>
</svg>

16
src/lib/icons/Plus.svelte Normal file
View File

@@ -0,0 +1,16 @@
<script lang="ts">
import type { SvelteHTMLElements } from 'svelte/elements';
interface Props extends Omit<SvelteHTMLElements['svg'], 'width' | 'height'> {
size?: string | number | null;
}
let { size = 24, ...rest }: Props = $props();
</script>
<svg width={size} height={size} viewBox="0 0 256 256" {...rest}>
<path
fill="currentColor"
d="M224,128a8,8,0,0,1-8,8H136v80a8,8,0,0,1-16,0V136H40a8,8,0,0,1,0-16h80V40a8,8,0,0,1,16,0v80h80A8,8,0,0,1,224,128Z"
/>
</svg>

20
src/lib/icons/Search.svelte Normal file
View File

@@ -0,0 +1,20 @@
<script lang="ts">
import type { SvelteHTMLElements } from 'svelte/elements';
interface Props extends Omit<SvelteHTMLElements['svg'], 'width' | 'height'> {
size?: string | number | null;
}
let { size = 24, ...rest }: Props = $props();
</script>
<svg viewBox="0 0 24 24" aria-hidden="true" width={size} height={size} {...rest} data-name="search">
<path
fill="none"
stroke="currentColor"
stroke-linecap="round"
stroke-linejoin="round"
stroke-width="2"
d="M3 10a7 7 0 1 0 14 0a7 7 0 1 0-14 0m18 11l-6-6"
/>
</svg>

16
src/lib/icons/Table.svelte Normal file
View File

@@ -0,0 +1,16 @@
<script lang="ts">
import type { SvelteHTMLElements } from 'svelte/elements';
interface Props extends Omit<SvelteHTMLElements['svg'], 'width' | 'height'> {
size?: string | number | null;
}
let { size = 24, ...rest }: Props = $props();
</script>
<svg width={size} height={size} viewBox="0 0 256 256" {...rest}>
<path
fill="currentColor"
d="M224,48H32a8,8,0,0,0-8,8V192a16,16,0,0,0,16,16H216a16,16,0,0,0,16-16V56A8,8,0,0,0,224,48ZM40,112H80v32H40Zm56,0H216v32H96ZM216,64V96H40V64ZM40,160H80v32H40Zm176,32H96V160H216v32Z"
/>
</svg>

3
src/lib/query.ts
View File

@@ -5,10 +5,9 @@ export async function exec(query: string) {
const r: string = await invoke('query', {
query
});
return JSON.parse(r);
return JSON.parse(r) as CHResponse;
} catch (e) {
console.error(e);
return {};
}
}

58
src/lib/schema.svelte
View File

@@ -1,58 +0,0 @@
<script lang="ts">
import { appWindow } from '@tauri-apps/api/window';
import { exec, type CHResponse } from './query';
import { onMount } from 'svelte';
let schema: CHResponse = $state.raw(undefined);
onMount(async () => {
schema = await exec(
`DESCRIBE TABLE s3('https://data.agnostic.dev/ethereum-mainnet-pq/logs/*.parquet', 'Parquet')`
);
});
appWindow.onFileDropEvent(async (event) => {
if (event.payload.type === 'drop') {
const path = event.payload.paths[0];
schema = await exec(`DESCRIBE '${path}'`);
}
});
</script>
<section>
{#if schema}
{#each schema.data as column}
<div class="column">
<span class="name">{column.name}</span>
<span class="type">{column.type}</span>
</div>
{/each}
{/if}
</section>
<style>
section {
padding: 10px;
background: hsl(0deg 0% 12%);
overflow: scroll;
font-size: 10px;
}
.column {
display: flex;
white-space: nowrap;
}
.name {
font-weight: 600;
width: 100px;
overflow: hidden;
text-overflow: ellipsis;
}
.type {
width: 100px;
overflow: hidden;
text-overflow: ellipsis;
}
</style>

16
src/lib/types.d.ts vendored Normal file
View File

@@ -0,0 +1,16 @@
export interface ColumnDescriptor {
name: string;
type: string;
}
export interface Dataset {
name: string;
/** Must be unique */
slug: string;
path: string;
type: 'CSV' | 'Parquet' | 'MergeTree';
/** Describe result */
columns?: ColumnDescriptor[];
/** Timestamp */
last_refresh: number;
}

12
src/lib/utils.ts
View File

@@ -14,15 +14,15 @@ export function getTextWidth(text: string) {
}
export function getRelativeParent<E extends Element>(node: E) {
let parent = node.parentElement
let parent = node.parentElement;
while (parent && getComputedStyle(parent).position !== 'relative') {
parent = parent.parentElement
}
while (parent && getComputedStyle(parent).position !== 'relative') {
parent = parent.parentElement;
}
return parent
return parent;
}
export function clamp(value: number, min: number, max: number) {
return Math.min(max, Math.max(value, min))
return Math.min(max, Math.max(value, min));
}

15
src/routes/+page.svelte
View File

@@ -3,8 +3,10 @@
import WindowTitleBar from '$lib/components/WindowTitleBar.svelte';
import { Editor } from '$lib/components/Editor';
import { exec, type CHResponse } from '$lib/query';
import Schema from '$lib/schema.svelte';
import SideBar from '$lib/components/SideBar.svelte';
import Result from '$lib/components/Result.svelte';
import type { Dataset } from '$lib/types';
import { getDefaultSource } from '$lib/components/Datasets/utils';
let response: CHResponse = $state.raw(undefined);
@@ -16,6 +18,15 @@
loading = true;
response = await exec(query).finally(() => (loading = false));
}
let sources = $state<Dataset[]>([]);
$effect.pre(() => {
if (!sources.length) {
getDefaultSource().then((source) => {
if (source) sources.push(source);
});
}
});
</script>
<WindowTitleBar>
@@ -27,7 +38,7 @@
<section class="screen">
<SplitPane orientation="horizontal" position="242px" min="242px" max="40%">
{#snippet a()}
<Schema />
<SideBar {sources} />
{/snippet}
{#snippet b()}
<SplitPane orientation="vertical" min="20%" max="80%" --color="hsl(0deg 0% 12%)">