windmill-components 1.473.1 → 1.493.3

package/package/components/S3FilePicker.svelte

@@ -1,4 +1,4 @@
-<script>import { File as FileIcon, FolderClosed, FolderOpen, RotateCw, Loader2, Download, Trash, FileUp, MoveRight } from 'lucide-svelte';
+<script lang="ts">import { File as FileIcon, FolderClosed, FolderOpen, RotateCw, Loader2, Download, Trash, FileUp, MoveRight } from 'lucide-svelte';
 import { workspaceStore } from '../stores';
 import { HelpersService } from '../gen';
 import { base } from '../base';
@@ -12,6 +12,7 @@ import TableSimple from './TableSimple.svelte';
 import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte';
 import FileUploadModal from './common/fileUpload/FileUploadModal.svelte';
 import PdfViewer from './display/PdfViewer.svelte';
+import { twMerge } from 'tailwind-merge';
 let deletionModalOpen = false;
 let fileDeletionInProgress = false;
 let fileListUnavailable = undefined;
@@ -26,6 +27,7 @@ export let initialFileKey = undefined;
 let initialFileKeyInternalCopy;
 export let selectedFileKey = undefined;
 export let folderOnly = false;
+export let regexFilter = undefined;
 let csvSeparatorChar = ',';
 let csvHasHeader = true;
 let dispatch = createEventDispatcher();
@@ -42,6 +44,7 @@ let listMarkers;
 let page = 0;
 const maxKeys = 1000;
 let count = 0;
+let displayedCount = 0;
 let filter = '';
 let timeout = undefined;
 let firstLoad = true;
@@ -50,20 +53,19 @@ function onFilterChange() {
     if (!firstLoad) {
         timeout && clearTimeout(timeout);
         timeout = setTimeout(() => {
-            page = 0;
-            listMarkers = [];
-            loadFiles();
+            clearAndLoadFiles({ keepFilter: true });
         }, 500);
     }
     else {
         firstLoad = false;
     }
 }
+let lastKeyFolders = [];
 async function loadFiles() {
     fileListLoading = true;
     let availableFiles = await HelpersService.listStoredFiles({
         workspace: $workspaceStore,
-        maxKeys: maxKeys,
+        maxKeys: maxKeys, // fixed pages of 1000 files for now
         marker: page == 0 ? undefined : listMarkers[page - 1],
         prefix: filter.trim() != '' ? filter : undefined,
         storage: storage
@@ -76,13 +78,18 @@ async function loadFiles() {
         return;
     }
     fileListUnavailable = false;
-    allFilesByKey = {};
-    displayedFileKeys = [];
-    for (let file_path of availableFiles.windmill_large_files) {
+    for (let [index, file_path] of availableFiles.windmill_large_files.entries()) {
+        if (regexFilter && !regexFilter.test(file_path.s3)) {
+            continue;
+        }
+        displayedCount += 1;
         let split_path = file_path.s3.split('/');
         let parent_path = undefined;
         let current_path = undefined;
         let nestingLevel = 0;
+        if (index === availableFiles.windmill_large_files.length - 1 && split_path.length > 1) {
+            lastKeyFolders = split_path.slice(0, -1);
+        }
         for (let i = 0; i < split_path.length; i++) {
             parent_path = current_path;
             current_path = current_path === undefined ? split_path[i] : current_path + split_path[i];
@@ -91,15 +98,17 @@ async function loadFiles() {
             }
             nestingLevel = i * 2;
             if (allFilesByKey[current_path] !== undefined) {
+                allFilesByKey[current_path].count += 1;
                 continue;
             }
             allFilesByKey[current_path] = {
                 type: i === split_path.length - 1 ? 'leaf' : 'folder',
                 full_key: current_path,
                 display_name: split_path[i],
-                collapsed: true,
+                collapsed: true, // folders collapsed by default
                 parentPath: parent_path,
-                nestingLevel: nestingLevel
+                nestingLevel: nestingLevel,
+                count: 1
             };
             if (i == 0) {
                 displayedFileKeys.push(current_path);
@@ -107,14 +116,13 @@ async function loadFiles() {
         }
     }
     if (listMarkers.length == page) {
-        count = availableFiles.windmill_large_files.length;
+        count += availableFiles.windmill_large_files.length;
         const nextMarker = availableFiles.windmill_large_files?.[availableFiles.windmill_large_files.length - 1]?.s3;
         if (nextMarker)
             listMarkers.push(nextMarker);
     }
-    displayedFileKeys = displayedFileKeys.sort();
     // before returning, un-collapse the folders containing the selected file (if any)
-    if (selectedFileKey !== undefined && !emptyString(selectedFileKey.s3)) {
+    if (selectedFileKey !== undefined && !emptyString(selectedFileKey.s3) && page === 0) {
         let split_path = selectedFileKey.s3.split('/');
         let current_path = undefined;
         for (let i = 0; i < split_path.length; i++) {
@@ -122,12 +130,19 @@ async function loadFiles() {
             if (i < split_path.length - 1) {
                 current_path += '/';
             }
-            let indexOf = displayedFileKeys.indexOf(current_path);
-            if (indexOf >= 0) {
-                selectItem(indexOf, true);
+            const folder = allFilesByKey[current_path];
+            if (folder) {
+                folder.collapsed = false;
+            }
+            for (let file_key in allFilesByKey) {
+                let file_info = allFilesByKey[file_key];
+                if (file_info.parentPath === current_path) {
+                    displayedFileKeys.push(file_key);
+                }
             }
         }
     }
+    displayedFileKeys = displayedFileKeys.sort();
     fileListLoading = false;
     fileInfoLoading = false;
 }
@@ -164,7 +179,7 @@ async function loadFilePreview(fileKey, fileSizeInBytes, fileMimeType) {
         csvSeparator: csvSeparatorChar,
         csvHasHeader: csvHasHeader,
         readBytesFrom: 0,
-        readBytesLength: 128 * 1024,
+        readBytesLength: 128 * 1024, // For now static limit of 128Kb per file,
         storage: storage
     });
     let filePreviewContent = filePreviewRaw.content;
@@ -212,13 +227,47 @@ async function deleteFileFromS3(fileKey) {
         deletionModalOpen = false;
     }
     sendUserToast(`${fileKey} deleted from S3 bucket`);
-    selectedFileKey = { s3: '' };
-    const idx = displayedFileKeys.indexOf(fileKey);
-    if (idx >= 0) {
-        displayedFileKeys.splice(idx, 1);
-        displayedFileKeys = [...displayedFileKeys];
+    selectedFileKey = { s3: '', storage };
+    const currentPage = page;
+    await clearAndLoadFiles();
+    for (let i = 0; i < currentPage; i++) {
+        page = i + 1;
+        await loadFiles();
+    }
+    const fileKeyFolders = fileKey.split('/').slice(0, -1);
+    let current_path = undefined;
+    for (let i = 0; i < fileKeyFolders.length; i++) {
+        current_path =
+            current_path === undefined ? fileKeyFolders[i] : current_path + fileKeyFolders[i];
+        if (i < fileKeyFolders.length) {
+            current_path += '/';
+        }
+        const folder = allFilesByKey[current_path];
+        if (folder) {
+            folder.collapsed = false;
+        }
+        for (let file_key in allFilesByKey) {
+            let file_info = allFilesByKey[file_key];
+            if (file_info.parentPath === current_path) {
+                displayedFileKeys.push(file_key);
+            }
+        }
+    }
+    displayedFileKeys = displayedFileKeys.sort();
+}
+async function clearAndLoadFiles({ keepFilter } = {}) {
+    displayedFileKeys = [];
+    allFilesByKey = {};
+    count = 0;
+    displayedCount = 0;
+    page = 0;
+    listMarkers = [];
+    fileMetadata = undefined;
+    filePreview = undefined;
+    if (!keepFilter) {
+        filter = '';
     }
-    delete allFilesByKey[fileKey];
+    await loadFiles();
 }
 async function moveS3File(srcFileKey, destFileKey) {
     fileMoveInProgress = true;
@@ -238,8 +287,8 @@ async function moveS3File(srcFileKey, destFileKey) {
         moveModalOpen = false;
     }
     sendUserToast(`${srcFileKey} moved to ${destFileKey}`);
-    selectedFileKey = { s3: destFileKey };
-    await loadFiles();
+    selectedFileKey = { s3: destFileKey, storage };
+    await clearAndLoadFiles();
     await loadFileMetadataPlusPreviewAsync(selectedFileKey.s3);
 }
 let storage = undefined;
@@ -249,14 +298,6 @@ export async function open(preSelectedFileKey = undefined) {
         initialFileKey = { ...preSelectedFileKey };
         selectedFileKey = { ...preSelectedFileKey };
     }
-    displayedFileKeys = [];
-    allFilesByKey = {};
-    count = 0;
-    page = 0;
-    filter = '';
-    listMarkers = [];
-    fileMetadata = undefined;
-    filePreview = undefined;
     reloadContent();
     drawer.openDrawer?.();
 }
@@ -264,6 +305,7 @@ async function reloadContent() {
     if (initialFileKey !== undefined) {
         initialFileKeyInternalCopy = { ...initialFileKey };
     }
+    fileListLoading = true;
     try {
         await HelpersService.datasetStorageTestConnection({
             workspace: $workspaceStore,
@@ -272,14 +314,15 @@ async function reloadContent() {
         workspaceSettingsInitialized = true;
     }
     catch (e) {
+        fileListLoading = false;
         console.error('Workspace not connected to object storage: ', e);
         workspaceSettingsInitialized = false;
         return;
     }
-    await loadFiles(); // TODO: Potentially load only on the first open and add a refresh button
+    await clearAndLoadFiles();
     if (selectedFileKey !== undefined) {
         if (allFilesByKey[selectedFileKey.s3] === undefined) {
-            selectedFileKey = { s3: '' };
+            selectedFileKey = { s3: '', storage };
         }
         else {
             loadFileMetadataPlusPreviewAsync(selectedFileKey.s3);
@@ -301,7 +344,8 @@ function selectItem(index, toggleCollapsed = true) {
     if (item.type === 'folder') {
         if (folderOnly) {
             selectedFileKey = {
-                s3: item_key
+                s3: item_key,
+                storage
             };
         }
         if (toggleCollapsed) {
@@ -339,7 +383,8 @@ function selectItem(index, toggleCollapsed = true) {
     }
     else {
         selectedFileKey = {
-            s3: item_key
+            s3: item_key,
+            storage
         };
         loadFileMetadataPlusPreviewAsync(selectedFileKey.s3);
     }
@@ -351,13 +396,20 @@ function selectItem(index, toggleCollapsed = true) {
 <Drawer
 	bind:this={drawer}
 	on:close={() => {
-		dispatch('close')
+		dispatch(
+			'close',
+			selectedFileKey?.s3
+				? {
+						s3: selectedFileKey.s3,
+						storage: storage
+					}
+				: undefined
+		)
 	}}
 	size="1200px"
 >
 	<DrawerContent
 		title="S3 file browser"
-		overflow_y={false}
 		on:close={exit}
 		tooltip="Files present in the Workspace S3 bucket. You can set the workspace S3 bucket in the settings."
 		documentationLink="https://www.windmill.dev/docs/integrations/s3"
@@ -409,16 +461,16 @@ function selectItem(index, toggleCollapsed = true) {
 			{/if}
 			<div class="flex flex-row border rounded-md h-full">
 				{#if !fileListUnavailable}
-					<div class="min-w-[30%] border-r h-full flex flex-col">
-						<div class="w-12/12 pb-2 flex flex-row mb-1 gap-1">
-							<input type="text" placeholder="Folder prefix" bind:value={filter} class="text-2xl" />
+					<div class="min-w-[30%] border-r flex flex-col">
+						<div class="w-full p-1 border-b">
+							<input type="text" placeholder="Folder prefix" bind:value={filter} class="text-xl" />
 						</div>
 						{#if fileListLoading === false && displayedFileKeys.length === 0}
 							<div class="p-4 text-tertiary text-xs text-center italic">
 								No files in the workspace S3 bucket at that prefix
 							</div>
 						{:else}
-							<div class="grow max-h-3/4" bind:clientHeight={listDivHeight}>
+							<div class="grow" bind:clientHeight={listDivHeight}>
 								<VirtualList
 									width="100%"
 									height={listDivHeight}
@@ -430,70 +482,76 @@ function selectItem(index, toggleCollapsed = true) {
 										let:index
 										let:style
 										{style}
-										class="hover:bg-surface-hover border"
+										class={twMerge(
+											'hover:bg-surface-hover border-b',
+											index === displayedFileKeys.length - 1 && 'border-b-0'
+										)}
 									>
 										{@const file_info = allFilesByKey[displayedFileKeys[index]]}
-										<div
-											on:click={() => selectItem(index)}
-											class={`flex flex-row h-full font-semibold text-xs items-center justify-start ${
-												selectedFileKey !== undefined && selectedFileKey.s3 === file_info.full_key
-													? 'bg-surface-hover'
-													: ''
-											} `}
-										>
+										{#if file_info}
 											<div
-												class={`flex flex-row w-full ml-${
-													2 + file_info.nestingLevel
-												} gap-2 h-full items-center`}
+												on:click={() => selectItem(index)}
+												class={twMerge(
+													'flex flex-row h-full font-semibold text-xs items-center justify-start',
+													selectedFileKey !== undefined && selectedFileKey.s3 === file_info.full_key
+														? 'bg-surface-hover'
+														: ''
+												)}
 											>
-												{#if file_info.type === 'folder'}
-													{#if file_info.collapsed}<FolderClosed size={16} />{:else}<FolderOpen
-															size={16}
-														/>{/if}
-													<div class="truncate text-ellipsis w-56">
-														{file_info.display_name}
-													</div>
-												{:else}
-													<FileIcon size={16} />
-													<div class="truncate text-ellipsis w-56">
-														{file_info.display_name}
-													</div>
-												{/if}
+												<div
+													class={`flex flex-row w-full gap-2 h-full items-center`}
+													style={`margin-left: ${(2 + file_info.nestingLevel) * 0.25}rem;`}
+												>
+													{#if file_info.type === 'folder'}
+														{#if file_info.collapsed}<FolderClosed size={16} />{:else}<FolderOpen
+																size={16}
+															/>{/if}
+														<div class="truncate text-ellipsis w-56">
+															{file_info.display_name} ({file_info.count}{count % 1000 === 0 &&
+															lastKeyFolders[file_info.nestingLevel / 2] === file_info.display_name
+																? '+'
+																: ''} item{file_info.count === 1 ? '' : 's'})
+														</div>
+													{:else}
+														<FileIcon size={16} />
+														<div class="truncate text-ellipsis w-56">
+															{file_info.display_name}
+														</div>
+													{/if}
+												</div>
 											</div>
-										</div>
-									</div></VirtualList
-								>
+										{/if}
+									</div>
+								</VirtualList>
 							</div>
 							<div
-								class="flex gap-2 text-2xs items-center text-secondary px-2 w-full h-max-[30px] pt-2 border-t"
+								class="flex flex-col gap-2 text-2xs justify-center items-center text-secondary w-full border-t h-16"
 							>
-								<div>{count} items on this page</div>
-								<div>Page {page + 1}</div>
+								{#if fileListLoading === true}
+									<div class="flex text-secondary mt-1 text-xs justify-center items-center w-full">
+										<Loader2 size={12} class="animate-spin mr-1" /> Loading content
+									</div>
+								{:else}
+									<div>
+										{displayedCount}{count % maxKeys === 0 ? '+' : ''}
+										{displayedCount !== count ? 'filtered ' : ''}items (including inside folders)
+									</div>
 
-								{#if count == maxKeys}
-									<button
-										class="text-secondary border p-1 underline text-2xs whitespace-nowrap text-center"
-										on:click={() => {
-											page -= 1
-											loadFiles()
-										}}
-										>Previous
-									</button>
-									<button
-										class="text-secondary border p-1 underline text-2xs whitespace-nowrap text-center"
-										on:click={() => {
-											page += 1
-											loadFiles()
-										}}
-										>Next
-									</button>
+									{#if count % maxKeys === 0}
+										<Button
+											variant="border"
+											color="light"
+											size="xs2"
+											on:click={() => {
+												page += 1
+												loadFiles()
+											}}
+										>
+											Load more
+										</Button>
+									{/if}
 								{/if}
 							</div>
-							{#if fileListLoading === true}
-								<div class="flex text-secondary mt-1 text-xs justify-center items-center w-full">
-									<Loader2 size={12} class="animate-spin mr-1" /> Loading content
-								</div>
-							{/if}
 						{/if}
 					</div>
 				{/if}
@@ -510,16 +568,16 @@ function selectItem(index, toggleCollapsed = true) {
 						</div>
 					{:else}
 						<div class="p-4 gap-2">
-							<Section label={fileMetadata.fileKey}>
+							<Section label={fileMetadata.fileKey} breakAll>
 								<div slot="action" class="flex gap-2">
 									{#if filePreview !== undefined}
 										<Button
 											title="Download file from S3"
 											variant="border"
 											color="light"
-											href={`${base}/api/w/${$workspaceStore}/job_helpers/download_s3_file?file_key=${
-												fileMetadata?.fileKey
-											}${storage ? `&storage=${storage}` : ''}`}
+											href={`${base}/api/w/${$workspaceStore}/job_helpers/download_s3_file?file_key=${encodeURIComponent(
+												fileMetadata?.fileKey ?? ''
+											)}${storage ? `&storage=${storage}` : ''}`}
 											download={fileMetadata?.fileKey.split('/').pop() ?? 'unnamed_download.file'}
 											startIcon={{ icon: Download }}
 											iconOnly={true}
@@ -557,77 +615,77 @@ function selectItem(index, toggleCollapsed = true) {
 					{/if}
 
 					<div class="flex flex-col h-full w-full overflow-auto text-xs p-4 bg-surface-secondary">
-						{#if fileMetadata?.fileKey.endsWith('.png') || fileMetadata?.fileKey.endsWith('.jpg') || fileMetadata?.fileKey.endsWith('.jpeg') || fileMetadata?.fileKey.endsWith('.webp')}
-							<div>
-								<img
-									src={`/api/w/${$workspaceStore}/job_helpers/load_image_preview?file_key=${fileMetadata.fileKey}` +
-										(storage ? `&storage=${storage}` : '')}
-									alt="S3 preview"
-								/>
-							</div>
-						{:else if fileMetadata?.fileKey.endsWith('.pdf')}
-							<div class="w-full h-[950px] border">
-								<PdfViewer
-									source={`/api/w/${$workspaceStore}/job_helpers/load_image_preview?file_key=${fileMetadata.fileKey}` +
-										(storage ? `&storage=${storage}` : '')}
-								/>
-							</div>
-						{:else if filePreviewLoading}
-							<div class="flex h-6 items-center text-tertiary mb-4">
-								<Loader2 size={12} class="animate-spin mr-1" /> File preview loading
-							</div>
-						{:else if fileMetadata !== undefined && filePreview !== undefined}
-							<div class="flex items-center text-tertiary mb-4">
-								{#if filePreview.contentType === 'Unknown'}
-									Type of file not supported for preview.
-								{:else if filePreview.contentType === 'Csv'}
-									Previewing a {filePreview.contentType?.toLowerCase()} file. Separator character:
-									<div class="inline-flex w-12 ml-2 mr-2">
-										<select
-											class="h-8"
-											bind:value={csvSeparatorChar}
-											on:change={(e) =>
-												loadFilePreview(
-													fileMetadata?.fileKey ?? '',
-													fileMetadata?.size,
-													fileMetadata?.mimeType
-												)}
-										>
-											<option value=",">,</option>
-											<option value=";">;</option>
-											<option value="\t">\t</option>
-											<option value="|">|</option>
-										</select>
-									</div>
-									Header row:
-									<div class="inline-flex item-center w-4 ml-2 mr-2">
-										<input
-											on:focus
-											on:click
-											disabled={false}
-											type="checkbox"
-											id="csv-header"
-											class="h-5"
-											bind:checked={csvHasHeader}
-											on:change|stopPropagation={(e) =>
-												loadFilePreview(
-													fileMetadata?.fileKey ?? '',
-													fileMetadata?.size,
-													fileMetadata?.mimeType
-												)}
-										/>
-									</div>
-								{:else}
-									Previewing a {filePreview.contentType?.toLowerCase()} file.
-								{/if}
-							</div>
-							<pre class="grow whitespace-no-wrap break-words">
-									{#if !emptyString(filePreview.contentPreview)}
-									{filePreview.contentPreview}
-								{:else if filePreview.contentType !== undefined}
-									Preview impossible.
-								{/if}
+						{#if filePreviewLoading || fileMetadata}
+							{#if fileMetadata?.fileKey.endsWith('.png') || fileMetadata?.fileKey.endsWith('.jpg') || fileMetadata?.fileKey.endsWith('.jpeg') || fileMetadata?.fileKey.endsWith('.webp')}
+								<div>
+									<img
+										src={`/api/w/${$workspaceStore}/job_helpers/load_image_preview?file_key=${encodeURIComponent(
+											fileMetadata.fileKey
+										)}` + (storage ? `&storage=${storage}` : '')}
+										alt="S3 preview"
+									/>
+								</div>
+							{:else if fileMetadata?.fileKey.endsWith('.pdf')}
+								<div class="w-full h-[950px] border">
+									<PdfViewer
+										source={`/api/w/${$workspaceStore}/job_helpers/load_image_preview?file_key=${encodeURIComponent(
+											fileMetadata.fileKey
+										)}` + (storage ? `&storage=${storage}` : '')}
+									/>
+								</div>
+							{:else if filePreviewLoading}
+								<div class="flex h-6 items-center text-tertiary mb-4">
+									<Loader2 size={12} class="animate-spin mr-1" /> File preview loading
+								</div>
+							{:else if fileMetadata !== undefined && filePreview !== undefined}
+								<div class="flex items-center text-tertiary mb-4">
+									{#if filePreview.contentType === 'Unknown'}
+										Type of file not supported for preview.
+									{:else if filePreview.contentType === 'Csv'}
+										Previewing a {filePreview.contentType?.toLowerCase()} file. Separator character:
+										<div class="inline-flex w-12 ml-2 mr-2">
+											<select
+												class="h-8"
+												bind:value={csvSeparatorChar}
+												on:change={(e) =>
+													loadFilePreview(
+														fileMetadata?.fileKey ?? '',
+														fileMetadata?.size,
+														fileMetadata?.mimeType
+													)}
+											>
+												<option value=",">,</option>
+												<option value=";">;</option>
+												<option value="\t">\t</option>
+												<option value="|">|</option>
+											</select>
+										</div>
+										Header row:
+										<div class="inline-flex item-center w-4 ml-2 mr-2">
+											<input
+												on:focus
+												on:click
+												disabled={false}
+												type="checkbox"
+												id="csv-header"
+												class="h-5"
+												bind:checked={csvHasHeader}
+												on:change|stopPropagation={(e) =>
+													loadFilePreview(
+														fileMetadata?.fileKey ?? '',
+														fileMetadata?.size,
+														fileMetadata?.mimeType
+													)}
+											/>
+										</div>
+									{:else}
+										Previewing a {filePreview.contentType?.toLowerCase()} file.
+									{/if}
+								</div>
+								<pre class="grow whitespace-no-wrap break-words"
+									>{#if !emptyString(filePreview.contentPreview)}{filePreview.contentPreview}{:else if filePreview.contentType !== undefined}Preview impossible.{/if}
 							</pre>
+							{/if}
 						{/if}
 					</div>
 				</div>
@@ -669,7 +727,7 @@ function selectItem(index, toggleCollapsed = true) {
 		deleteFileFromS3(fileMetadata?.fileKey)
 	}}
 	keyListen={false}
-	bind:loading={fileDeletionInProgress}
+	loading={fileDeletionInProgress}
 >
 	<div class="flex flex-col w-full space-y-4">
 		<span
@@ -689,7 +747,7 @@ function selectItem(index, toggleCollapsed = true) {
 		moveS3File(fileMetadata?.fileKey, moveDestKey)
 	}}
 	keyListen={false}
-	bind:loading={fileMoveInProgress}
+	loading={fileMoveInProgress}
 >
 	<div class="flex flex-col space-y-4">
 		<div class="flex items-center justify-between">
@@ -711,8 +769,8 @@ function selectItem(index, toggleCollapsed = true) {
 	on:close={async (evt) => {
 		uploadModalOpen = false
 		if (evt.detail !== undefined && evt.detail !== null) {
-			selectedFileKey = { s3: evt.detail }
-			loadFiles()
+			selectedFileKey = { s3: evt.detail, storage }
+			await clearAndLoadFiles()
 			loadFileMetadataPlusPreviewAsync(evt.detail)
 		}
 	}}