From 95034f0a1caf7ada079f78746cba0a16251318e8 Mon Sep 17 00:00:00 2001 From: Superchupu <53496941+SuperchupuDev@users.noreply.github.com> Date: Tue, 6 Aug 2024 05:36:18 +0200 Subject: [PATCH 1/8] chore: exclude irrelevant files from published bundle (#105) --- __tests__/fdir.test.ts | 2 +- benchmarks/benchmark.js | 2 +- index.ts | 4 ---- src/index.ts | 5 +++++ tsconfig.json | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) delete mode 100644 index.ts diff --git a/__tests__/fdir.test.ts b/__tests__/fdir.test.ts index 882e2afd..26ec1c92 100644 --- a/__tests__/fdir.test.ts +++ b/__tests__/fdir.test.ts @@ -1,4 +1,4 @@ -import { fdir } from "../index"; +import { fdir } from "../src/index"; import fs from "fs"; import mock from "mock-fs"; import { test, beforeEach, TestContext } from "vitest"; diff --git a/benchmarks/benchmark.js b/benchmarks/benchmark.js index 54c8de2c..4edfda7f 100644 --- a/benchmarks/benchmark.js +++ b/benchmarks/benchmark.js @@ -1,4 +1,4 @@ -import { fdir } from "../index"; +import { fdir } from "../src/index"; import { fdir as fdir5 } from "fdir5"; import { fdir as fdir4 } from "fdir4"; import fdir3 from "fdir3"; diff --git a/index.ts b/index.ts deleted file mode 100644 index a90bd190..00000000 --- a/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { Builder } from "./src/builder"; -export { Builder as fdir }; -export type Fdir = typeof Builder; -export * from "./src"; diff --git a/src/index.ts b/src/index.ts index eea524d6..fba8ef8b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1 +1,6 @@ +import { Builder } from "./builder"; + +export { Builder as fdir }; +export type Fdir = typeof Builder; + export * from "./types"; diff --git a/tsconfig.json b/tsconfig.json index 0e8c8229..e285fa9b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -13,5 +13,5 @@ "resolveJsonModule": true, "allowJs": true }, - "include": ["./src", "./index.ts", "./benchmarks/"] + "include": ["./src"] } From e084ad2f2d5f1d60b6b1e80c503dfcf5a60fbd33 Mon Sep 17 00:00:00 2001 From: Jonathan Jogenfors Date: Sun, 25 Aug 2024 09:45:47 +0200 Subject: [PATCH 2/8] docs: remove Immich CLI (#108) The legacy Immich CLI is no longer available, remove link. The new Immich CLI uses another library --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 51243dd3..9accdf07 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,7 @@ Please check the benchmark against the latest version [here](/BENCHMARKS.md). 6. [netlify/build](https://github.com/netlify/build) 7. [FredKSchott/snowpack](https://github.com/FredKSchott/snowpack)\* 8. [yassinedoghri/astro-i18next](https://github.com/yassinedoghri/astro-i18next) -9. [immich-app/CLI](https://github.com/immich-app/CLI) -10. [selfrefactor/rambda](https://github.com/selfrefactor/rambda) +9. [selfrefactor/rambda](https://github.com/selfrefactor/rambda) - `snowpack` has since been discontinued. From 307c64167418bdd45dc08ba9b00d0ab84d1d699d Mon Sep 17 00:00:00 2001 From: Superchupu <53496941+SuperchupuDev@users.noreply.github.com> Date: Sun, 25 Aug 2024 08:47:00 +0100 Subject: [PATCH 3/8] docs: document missing options (#110) --- documentation.md | 50 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/documentation.md b/documentation.md index 54f8a47c..689f37b4 100644 --- a/documentation.md +++ b/documentation.md @@ -134,6 +134,16 @@ Use this to limit the maximum depth fdir will crawl to before stopping. const crawler = new fdir().withMaxDepth(5); ``` +### `withMaxFiles(number)` + +Use this to limit the maximum number of files fdir will crawl to before stopping. + +**Usage** + +```js +const crawler = new fdir().withMaxFiles(100); +``` + ### `withFullPaths` Use this to get full absolute paths in the output. @@ -156,6 +166,28 @@ Use this to get paths relative to the root directory in the output. const crawler = new fdir().withRelativePaths(); ``` +### `withPathSeparator` + +Use this to set the path separator in the output. + +**Usage** + +```js +const crawler = new fdir().withPathSeparator("/"); +``` + +### `withAbortSignal(AbortSignal)` + +Use this to pass an `AbortSignal` to the crawler. + +**Usage** + +```js +const controller = new AbortController(); + +const crawler = new fdir().withAbortSignal(controller.signal); +``` + ### `withErrors` Use this if you want to handle all errors manually. @@ -241,6 +273,19 @@ Applies a `glob` filter to all files and only adds those that satisfy it. const crawler = new fdir().glob("./**/*.js", "./**/*.md"); ``` +### `globWithOptions(string[], Object)` + +The same as `glob` but allows you to pass options to the matcher. + +**Usage** + +```js +// only get js and md files +const crawler = new fdir().globWithOptions(["**/*.js", "**/*.md"], { + strictSlashes: true +}); +``` + ### `filter(Function)` Applies a filter to all directories and files and only adds those that satisfy the filter. @@ -373,14 +418,17 @@ type Options = { includeDirs?: boolean; normalizePath?: boolean; maxDepth?: number; + maxFiles?: number; resolvePaths?: boolean; suppressErrors?: boolean; group?: boolean; onlyCounts?: boolean; - filters?: FilterFn[]; + filters: FilterFn[]; resolveSymlinks?: boolean; excludeFiles?: boolean; exclude?: ExcludeFn; relativePaths?: boolean; + pathSeparator: PathSeparator; + signal?: AbortSignal; }; ``` From fe59dd97b6f0c52786768d010c6194393e92d1dd Mon Sep 17 00:00:00 2001 From: Superchupu <53496941+SuperchupuDev@users.noreply.github.com> Date: Sun, 25 Aug 2024 08:47:23 +0100 Subject: [PATCH 4/8] chore: fix benchmark script (#111) --- benchmarks/fdir-benchmark.ts | 4 ++-- benchmarks/glob-benchmark.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/fdir-benchmark.ts b/benchmarks/fdir-benchmark.ts index 93f2a0d5..09004ad5 100644 --- a/benchmarks/fdir-benchmark.ts +++ b/benchmarks/fdir-benchmark.ts @@ -1,5 +1,5 @@ import child_process from "child_process"; -import { Fdir } from "../index"; +import { Fdir } from "../src/index"; import b from "benny"; type Version = typeof versions[number] | "current"; @@ -32,7 +32,7 @@ function addSuite(instance: Fdir, version: Version) { } async function fillSuites() { - const { fdir } = await import("../index"); + const { fdir } = await import("../src/index"); addSuite(fdir, "current"); versions.forEach((version) => { diff --git a/benchmarks/glob-benchmark.ts b/benchmarks/glob-benchmark.ts index d26f6bb8..acaac5da 100644 --- a/benchmarks/glob-benchmark.ts +++ b/benchmarks/glob-benchmark.ts @@ -1,4 +1,4 @@ -import { fdir } from "../index"; +import { fdir } from "../src/index"; import { glob, globSync } from "glob"; import fg from "fast-glob"; import tg from "tiny-glob"; From f425d11f3b291332754f2bd6e137615b91707342 Mon Sep 17 00:00:00 2001 From: Superchupu <53496941+SuperchupuDev@users.noreply.github.com> Date: Sun, 25 Aug 2024 09:01:31 +0100 Subject: [PATCH 5/8] feat: restore `withSymlinks` method (#104) * feat: add `useRealPaths` back * chore: update checks Co-authored-by: Abdullah Atta * chore: use `lstat` instead * Revert "chore: use `lstat` instead" This reverts commit e6ebb4410be9121d1cb1962159822c1f07ac30a5. --------- Co-authored-by: Abdullah Atta --- __tests__/fdir.test.ts | 36 +++++++++++++++++++++++ documentation.md | 7 +++-- src/api/functions/resolve-symlink.ts | 43 +++++++++++++++++++++++++++- src/builder/index.ts | 3 +- src/types.ts | 1 + 5 files changed, 85 insertions(+), 5 deletions(-) diff --git a/__tests__/fdir.test.ts b/__tests__/fdir.test.ts index 26ec1c92..7d60dcb7 100644 --- a/__tests__/fdir.test.ts +++ b/__tests__/fdir.test.ts @@ -322,6 +322,21 @@ for (const type of apiTypes) { mock.restore(); }); + test(`[${type}] crawl all files and include resolved symlinks without real paths`, async (t) => { + mock(mockFsWithSymlinks); + + const api = new fdir().withSymlinks({ resolvePaths: false }).crawl("/some/dir"); + const files = await api[type](); + t.expect(files).toHaveLength(3); + t.expect( + files.indexOf(resolveSymlinkRoot("/some/dir/dirSymlink/file-1")) > -1 + ).toBeTruthy(); + t.expect( + files.indexOf(resolveSymlinkRoot("/some/dir/dirSymlink/file-excluded-1")) > -1 + ).toBeTruthy(); + mock.restore(); + }); + test("crawl all files and include resolved symlinks with exclusions", async (t) => { mock(mockFsWithSymlinks); const api = new fdir() @@ -379,6 +394,27 @@ for (const type of apiTypes) { } mock.restore(); }); + + test("crawl all files (including symlinks without real paths) and throw errors", async (t) => { + mock({ + "/other/dir": {}, + "/some/dir": { + fileSymlink: mock.symlink({ + path: "/other/dir/file-3", + }), + }, + }); + + try { + const api = new fdir().withErrors().withSymlinks({ resolvePaths: false }).crawl("/some/dir"); + + await api[type](); + } catch (e) { + if (e instanceof Error) + t.expect(e.message.includes("no such file or directory")).toBeTruthy(); + } + mock.restore(); + }); } test(`[async] crawl directory & use abort signal to abort`, async (t) => { diff --git a/documentation.md b/documentation.md index 689f37b4..1a5ee840 100644 --- a/documentation.md +++ b/documentation.md @@ -102,7 +102,7 @@ Use this to also add the directories to the output. const crawler = new fdir().withDirs(); ``` -### `withSymlinks(boolean)` +### `withSymlinks({ resolvePaths: boolean })` Use this to follow all symlinks recursively. @@ -116,10 +116,10 @@ Use this to follow all symlinks recursively. ```js // to resolve all symlinked paths to their original path -const crawler = new fdir().withSymlinks(true); +const crawler = new fdir().withSymlinks({ resolvePaths: true }); // to disable path resolution -const crawler = new fdir().withSymlinks(false); +const crawler = new fdir().withSymlinks({ resolvePaths: false }); ``` ### `withMaxDepth(number)` @@ -425,6 +425,7 @@ type Options = { onlyCounts?: boolean; filters: FilterFn[]; resolveSymlinks?: boolean; + useRealPaths?: boolean; excludeFiles?: boolean; exclude?: ExcludeFn; relativePaths?: boolean; diff --git a/src/api/functions/resolve-symlink.ts b/src/api/functions/resolve-symlink.ts index 2a91d5b3..e35c3110 100644 --- a/src/api/functions/resolve-symlink.ts +++ b/src/api/functions/resolve-symlink.ts @@ -18,6 +18,28 @@ const resolveSymlinksAsync: ResolveSymlinkFunction = function( } = state; queue.enqueue(); + fs.stat(path, (error, stat) => { + if (error) { + queue.dequeue(suppressErrors ? null : error, state); + return; + } + + callback(stat, path); + queue.dequeue(null, state); + }); +}; + +const resolveSymlinksWithRealPathsAsync: ResolveSymlinkFunction = function( + path, + state, + callback +) { + const { + queue, + options: { suppressErrors }, + } = state; + queue.enqueue(); + fs.realpath(path, (error, resolvedPath) => { if (error) { queue.dequeue(suppressErrors ? null : error, state); @@ -36,6 +58,19 @@ const resolveSymlinksSync: ResolveSymlinkFunction = function( path, state, callback +) { + try { + const stat = fs.statSync(path); + callback(stat, path); + } catch (e) { + if (!state.options.suppressErrors) throw e; + } +}; + +const resolveSymlinksWithRealPathsSync: ResolveSymlinkFunction = function( + path, + state, + callback ) { try { const resolvedPath = fs.realpathSync(path); @@ -52,5 +87,11 @@ export function build( ): ResolveSymlinkFunction | null { if (!options.resolveSymlinks) return null; - return isSynchronous ? resolveSymlinksSync : resolveSymlinksAsync; + if (options.useRealPaths) + return isSynchronous + ? resolveSymlinksWithRealPathsSync + : resolveSymlinksWithRealPathsAsync; + return isSynchronous + ? resolveSymlinksSync + : resolveSymlinksAsync; } diff --git a/src/builder/index.ts b/src/builder/index.ts index a4fabb6b..2bcf8042 100644 --- a/src/builder/index.ts +++ b/src/builder/index.ts @@ -80,8 +80,9 @@ export class Builder { return this; } - withSymlinks() { + withSymlinks({ resolvePaths = true } = {}) { this.options.resolveSymlinks = true; + this.options.useRealPaths = resolvePaths; return this.withFullPaths(); } diff --git a/src/types.ts b/src/types.ts index e2bc00df..3c5ccf9c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -53,6 +53,7 @@ export type Options = { onlyCounts?: boolean; filters: FilterPredicate[]; resolveSymlinks?: boolean; + useRealPaths?: boolean; excludeFiles?: boolean; exclude?: ExcludePredicate; relativePaths?: boolean; From 646e47c1468b19c7ecd0e305588440285f56d6fa Mon Sep 17 00:00:00 2001 From: Abdullah Atta Date: Sun, 25 Aug 2024 13:15:56 +0500 Subject: [PATCH 6/8] test: fix try-catch assertion tests --- __tests__/fdir.test.ts | 69 ++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 37 deletions(-) diff --git a/__tests__/fdir.test.ts b/__tests__/fdir.test.ts index 7d60dcb7..4a2d3960 100644 --- a/__tests__/fdir.test.ts +++ b/__tests__/fdir.test.ts @@ -192,12 +192,8 @@ for (const type of apiTypes) { }); test(`[${type}] getting files from restricted directory should throw`, async (t) => { - try { - const api = new fdir().withErrors().crawl(restricted()); - await api[type](); - } catch (e) { - t.expect(e).toBeTruthy(); - } + const api = new fdir().withErrors().crawl(restricted()); + t.expect(async () => await api[type]()).rejects.toThrowError(); }); test(`[${type}] getting files from restricted directory shouldn't throw (suppressErrors)`, async (t) => { @@ -281,16 +277,6 @@ for (const type of apiTypes) { t.expect(result).toHaveLength(2); }); - test(`[${type}] giving undefined directory path should throw`, async (t) => { - // @ts-ignore - const api = new fdir().crawl(); - try { - await api[type](); - } catch (e) { - t.expect(!!e).toBeTruthy(); - } - }); - test(`[${type}] crawl and return relative paths`, async (t) => { const api = new fdir() .withRelativePaths() @@ -325,14 +311,18 @@ for (const type of apiTypes) { test(`[${type}] crawl all files and include resolved symlinks without real paths`, async (t) => { mock(mockFsWithSymlinks); - const api = new fdir().withSymlinks({ resolvePaths: false }).crawl("/some/dir"); + const api = new fdir() + .withSymlinks({ resolvePaths: false }) + .crawl("/some/dir"); const files = await api[type](); t.expect(files).toHaveLength(3); t.expect( files.indexOf(resolveSymlinkRoot("/some/dir/dirSymlink/file-1")) > -1 ).toBeTruthy(); t.expect( - files.indexOf(resolveSymlinkRoot("/some/dir/dirSymlink/file-excluded-1")) > -1 + files.indexOf( + resolveSymlinkRoot("/some/dir/dirSymlink/file-excluded-1") + ) > -1 ).toBeTruthy(); mock.restore(); }); @@ -374,9 +364,11 @@ for (const type of apiTypes) { t.expect(files.every((f) => !f.includes(sep))).toBeTruthy(); }); - test("crawl all files (including symlinks) and throw errors", async (t) => { + test(`[${type}] crawl all files (including symlinks)`, async (t) => { mock({ - "/other/dir": {}, + "/other/dir": { + "file-3": "somefile", + }, "/some/dir": { fileSymlink: mock.symlink({ path: "/other/dir/file-3", @@ -384,20 +376,19 @@ for (const type of apiTypes) { }, }); - try { - const api = new fdir().withErrors().withSymlinks().crawl("/some/dir"); - - await api[type](); - } catch (e) { - if (e instanceof Error) - t.expect(e.message.includes("no such file or directory")).toBeTruthy(); - } + const api = new fdir().withErrors().withSymlinks().crawl("/some/dir"); + const files = await api[type](); + t.expect( + files.indexOf(resolveSymlinkRoot("/other/dir/file-3")) > -1 + ).toBeTruthy(); mock.restore(); }); - test("crawl all files (including symlinks without real paths) and throw errors", async (t) => { + test(`[${type}] crawl all files (including symlinks without real paths)`, async (t) => { mock({ - "/other/dir": {}, + "/other/dir": { + "file-3": "somefile", + }, "/some/dir": { fileSymlink: mock.symlink({ path: "/other/dir/file-3", @@ -405,14 +396,18 @@ for (const type of apiTypes) { }, }); - try { - const api = new fdir().withErrors().withSymlinks({ resolvePaths: false }).crawl("/some/dir"); + const api = new fdir() + .withErrors() + .withSymlinks({ resolvePaths: false }) + .crawl("/some/dir"); + + await api[type](); + + const files = await api[type](); + t.expect( + files.indexOf(resolveSymlinkRoot("/some/dir/fileSymlink/")) > -1 + ).toBeTruthy(); - await api[type](); - } catch (e) { - if (e instanceof Error) - t.expect(e.message.includes("no such file or directory")).toBeTruthy(); - } mock.restore(); }); } From 8e3976b76dea86c6b50e04c10f0b4445c567779a Mon Sep 17 00:00:00 2001 From: Abdullah Atta Date: Sun, 25 Aug 2024 18:40:54 +0500 Subject: [PATCH 7/8] fix: file symlink path shouldn't end with slash --- __tests__/fdir.test.ts | 2 +- src/api/functions/resolve-symlink.ts | 20 +++++++++----------- src/api/walker.ts | 6 +----- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/__tests__/fdir.test.ts b/__tests__/fdir.test.ts index 4a2d3960..39481e62 100644 --- a/__tests__/fdir.test.ts +++ b/__tests__/fdir.test.ts @@ -405,7 +405,7 @@ for (const type of apiTypes) { const files = await api[type](); t.expect( - files.indexOf(resolveSymlinkRoot("/some/dir/fileSymlink/")) > -1 + files.indexOf(resolveSymlinkRoot("/some/dir/fileSymlink")) > -1 ).toBeTruthy(); mock.restore(); diff --git a/src/api/functions/resolve-symlink.ts b/src/api/functions/resolve-symlink.ts index e35c3110..334050ee 100644 --- a/src/api/functions/resolve-symlink.ts +++ b/src/api/functions/resolve-symlink.ts @@ -7,7 +7,7 @@ export type ResolveSymlinkFunction = ( callback: (stat: fs.Stats, path: string) => void ) => void; -const resolveSymlinksAsync: ResolveSymlinkFunction = function( +const resolveSymlinksAsync: ResolveSymlinkFunction = function ( path, state, callback @@ -24,12 +24,12 @@ const resolveSymlinksAsync: ResolveSymlinkFunction = function( return; } - callback(stat, path); - queue.dequeue(null, state); + callback(stat, path); + queue.dequeue(null, state); }); }; -const resolveSymlinksWithRealPathsAsync: ResolveSymlinkFunction = function( +const resolveSymlinksWithRealPathsAsync: ResolveSymlinkFunction = function ( path, state, callback @@ -54,7 +54,7 @@ const resolveSymlinksWithRealPathsAsync: ResolveSymlinkFunction = function( }); }; -const resolveSymlinksSync: ResolveSymlinkFunction = function( +const resolveSymlinksSync: ResolveSymlinkFunction = function ( path, state, callback @@ -67,7 +67,7 @@ const resolveSymlinksSync: ResolveSymlinkFunction = function( } }; -const resolveSymlinksWithRealPathsSync: ResolveSymlinkFunction = function( +const resolveSymlinksWithRealPathsSync: ResolveSymlinkFunction = function ( path, state, callback @@ -89,9 +89,7 @@ export function build( if (options.useRealPaths) return isSynchronous - ? resolveSymlinksWithRealPathsSync - : resolveSymlinksWithRealPathsAsync; - return isSynchronous - ? resolveSymlinksSync - : resolveSymlinksAsync; + ? resolveSymlinksWithRealPathsSync + : resolveSymlinksWithRealPathsAsync; + return isSynchronous ? resolveSymlinksSync : resolveSymlinksAsync; } diff --git a/src/api/walker.ts b/src/api/walker.ts index 338316d5..be1f65ee 100644 --- a/src/api/walker.ts +++ b/src/api/walker.ts @@ -117,11 +117,7 @@ export class Walker { if (exclude && exclude(entry.name, path)) continue; this.walkDirectory(this.state, path, depth - 1, this.walk); } else if (entry.isSymbolicLink() && resolveSymlinks) { - let path = joinPath.joinDirectoryPath( - entry.name, - directoryPath, - this.state.options.pathSeparator - ); + let path = this.joinPath(entry.name, directoryPath); this.resolveSymlink!(path, this.state, (stat, resolvedPath) => { if (stat.isDirectory()) { resolvedPath = this.normalizePath(resolvedPath); From 5a0aab5b3e108eaf677e0e39c22ff1ea03cd1a59 Mon Sep 17 00:00:00 2001 From: Abdullah Atta Date: Sun, 25 Aug 2024 18:53:21 +0500 Subject: [PATCH 8/8] chore: bump version to 6.3.0 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0cc9f797..c1040a07 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "fdir", - "version": "6.2.0", + "version": "6.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "fdir", - "version": "6.2.0", + "version": "6.3.0", "license": "MIT", "devDependencies": { "@types/glob": "^8.1.0", diff --git a/package.json b/package.json index 15ea5333..4a93d33c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "fdir", - "version": "6.2.0", + "version": "6.3.0", "description": "The fastest directory crawler & globbing alternative to glob, fast-glob, & tiny-glob. Crawls 1m files in < 1s", "main": "dist/index.js", "types": "dist/index.d.ts",