nv-file-vfs-local-alone
v1.0.27
Published
nv-file-vfs-local ================ - simple virtual file AND with local file persistence , only add/write/modify/del will update with local fs
Downloads
8
Readme
nv-file-vfs-local-alone
simple virtual file AND with local file persistence , only add/write/modify/del will update with local fs
alone mode is for test local-fs performance
support 13 actions:
rename(nvpath,name) |
rename_self(name)
cd(nvpath) |
ls(nvpath) |
tree(nvpath) |
touch(nvpath) |
mkdir(nvpath) |
ln(linkto_nvpath,linkfrom_nvpath) /softlink ln -s/ |
rm(nvpath) |
erase(nvpath) /removed AND cant recycle/ |
swap(nvpath0,nvpath1) |
mv(src_nvpath,dst_nvpath)
cp(src_nvpath,dst_nvpath)
nvpath is a path format:
begin with "/" is absolute path
end with "/" is a dir
"../" means parent ; ".../" means parent-of-parent ; "..../" "...../" ....
suitable for small scale datas( <= 1000000) in local fs
its a simplified version of nv-remote-file, remove binding of fanotify AND make it pure JS
install
main
npm install nv-file-vfs-localtool
npm install nv-file-vfs-local-cli -g
# nv_vfs_lcl_show -h
Usage: nv_vfs_lcl_show [options]
Options:
-w, --workdir workdir default "./"
-i, --id fnode id
-h, --help usageusage
const {
Disk,FileNode,
load_disk_from_single_file_dump, //fast
load_disk_from_workdir, //very-slow
load_disk_from_real_dir, //very-very-slow
} = require("nv-file-vfs-local-alone");
example
init virtual disk (just a empty dir with a meta-json file)
var disk = new Disk(20000);
await disk.init("TEST/tree");
/*
> disk
Disk {
fid: 'NlRbudHlLHMHQUjtJsAfGkKDdsBjffmYiegc',
max_size: 20000,
idpool: { minid_: 1, maxid_: 20000, used_: 0, lefted_: 20000 }
}
> disk.fdisk()
[]
>
nv-file-vfs-local-alone# tree TEST/tree/
TEST/tree/
└── ___disk___
├── ___data___
└── ___meta___.json
2 directories, 1 file
nv-file-vfs-local-alone#
nv-file-vfs-local-alone# cat TEST/tree/___disk___/___meta___.json
["NlRbudHlLHMHQUjtJsAfGkKDdsBjffmYiegc",20000,"v8",true]
fid@[0] : its a unique id , for remote-async/version using, USELESS in this pkg,
20000@[1]: max_size count(file+dir+slink) <= max_size, can NOT change after init, make sure it <=2**18 (262144), coz performance issue of local fs api
v8@[2]: means its serialized with v8.serialize, v8|json is supported, default is v8 for structured-clone
true@[3]: means its compressed by brotli, default true
*/creat root
var rt = await disk.creat_root_dir();
/*
> rt
{} %NlRbudHl:1% {}
>
> rt.is_dir()
true
> rt.nvpath_ //root nvpath is always '/'
'/'
> rt.$_raw_tag_ //root tag is always ''
''
> rt.$tag_ // this is a getter its a getter , only for display, {} means its a empty-leaf-dir
'{}'
>
*/
/*
nv-file-vfs-local-alone# tree TEST/tree/
TEST/tree/
└── ___disk___
├── ___data___
│ └── 1 // file-name 1 is the-unique-fnode-id, it is always unique in one disk
└── ___meta___.json // but NOT always continuous increase
2 directories, 2 files
nv-file-vfs-local-alone# nv_vfs_lcl_show -w TEST/tree/ -id 1
[
0, 0, 0, 0, 0, //#0/ the first 5 number is for internal using , it keep and track the relation of fnodes
1, //#1/ this is fnode-type
0: 'unknown', --should NOT be unknown ,unknown means its wait for fs resolve,
--OR NOT inited
1: 'dir',
2: 'file',
3: 'slink', --soft-link
'', //#2/ this is raw-tag/name
{}, //#3/ this is deserialized-data that the fnode contained, can NOT be func/cls/gen/promise..
which cant be serialized by nodejs
undefined, //#4/ this is linked-to-fnode if fnode-type===3<soft-link>
undefined means its a file or dir
Set(0) {} //#5/ linked-from-nodes , for tracking to prevent circular soft-link
]
*/
mkdir mkdir OR mkdir -p
var c = await rt.mkdir("a/b/c");
/*
> rt
%NlRbudHl:1% {} [
└── a
└── b
└── c{}
]
>
*/
/*
nv-file-vfs-local# tree TEST/
TEST/
└── tree
└── ___disk___
├── ___data___
│ ├── 1 //root
│ ├── 2 //a
│ ├── 3 //b
│ └── 4 //c
└── ___meta___.json
3 directories, 5 files
nv-file-vfs-local#
*/cd
//only add AND modify action is async , read-like action(cd,read...) is sync
var a = rt.cd("a");
var b = rt.cd("a/b");
/*
> a
a %NlRbudHl:2% {} [
a
└── b
└── c{}
]
> b
b %NlRbudHl:3% {} [
b
└── c{}
]
>
*/touch mkdir-p + touch
- if end with "/" , it is a dir; else it is a file
var d = await c.touch("d/dd/ddd");
/*
> rt
%NlRbudHl:1% {} [
└── a
└── b
└── c
└── d [...] //max-display-depth is 4 , coz console.log performance very bad
]
>
> rt.cd("a/b/c/d")
d %NlRbudHl:5% {} [
d
└── dd
└── ddd
]
>
*/
/*
nv-file-vfs-local# tree TEST/
TEST/
└── tree
└── ___disk___
├── ___data___
│ ├── 1 //root
│ ├── 2 //a
│ ├── 3 //b
│ ├── 4 //c
│ ├── 5 //d
│ ├── 6 //dd
│ └── 7 //ddd
└── ___meta___.json
3 directories, 8 files
nv-file-vfs-local#
*/
/*
> rt.$sdfs_.map(r=>r.nvpath_)
[
'/',
'/a/',
'/a/b/',
'/a/b/c/',
'/a/b/c/d/',
'/a/b/c/d/dd/',
'/a/b/c/d/dd/ddd'
]
>
*/ var e = await c.touch("e");
// methods begin with $ is used to find tree-relationship, many, tab to see them
// $methods end with _ is getter/setter
// $methods begin with $gen is generator
/*
> e.$parent_
c %NlRbudHl:4% {} [
c
├── d
│ └── dd
│ └── ddd
└── e
]
> e.$lsib_
d %NlRbudHl:5% {} [
d
└── dd
└── ddd
]
> c.cd("d/dd").$runcle_
e %NlRbudHl:8% {}
>
*/
/*
nv-file-vfs-local-alone# tree TEST/tree/
TEST/tree/
└── ___disk___
├── ___data___
│ ├── 1
│ ├── 2
│ ├── 3
│ ├── 4
│ ├── 5
│ ├── 6
│ ├── 7
│ └── 8 //---- e
└── ___meta___.json
2 directories, 9 files
*/ln -s ln(linked_to,linked_from) linked_to must exist
var Y = await rt.ln("a/b/c/e","X/Y");
/*
> rt
%NlRbudHl:1% {} [
├── a
│ └── b
│ └── c
│ ├── d [...]
│ └── e
└── X
└── Y -> /a/b/c/e
]
>
> Y.ref_ === e //soft-link to
true // each soft-link-fnode cant ONLY link to ONE other fnode
// coz multi inlets/outlets will make the structure to a Graph, that is too complicated
>
> Y.refid_
8
> e.$id_
8
>
> e.refed_by_ //linked-from-fnodeS , can more-than-one
Set(1) { Y -> /a/b/c/e %NlRbudHl:10% {} }
>
> e.refed_by_ids_
[ 10 ]
>
> Y.$id_
10
>
var e = b.cd("c/e");
*/
/*
nv-file-vfs-local# tree TEST/
TEST/
└── tree
└── ___disk___
├── ___data___
│ ├── 1
│ ├── 10 //Y
│ ├── 2
│ ├── 3
│ ├── 4
│ ├── 5
│ ├── 6
│ ├── 7
│ ├── 8
│ └── 9 //X
└── ___meta___.json
3 directories, 11 files
nv-file-vfs-local#
*/write !!must USE 【awrite】which begin with "a" !! , write is a USELESS method just for test,write will not do persistence
await c.awrite({meta:6666,data:"abcdefg"});
/*
> c
c %NlRbudHl:4% {
"meta": 6666,
"data": "abcdefg"
} [
c
├── d
│ └── dd
│ └── ddd
└── e
]
>
*/
/*
nv-file-vfs-local-alone# nv_vfs_lcl_show -w TEST/tree/ -id 4
[
5,
0,
3,
0,
8,
1,
'c',
{ meta: 6666, data: 'abcdefg' },
undefined,
Set(0) {}
]
nv-file-vfs-local-alone#
*/
rename
self
await b.rename_self("BABABA");
/*
> await b.rename_self("BABABA");
BABABA %NlRbudHl:3% {} [
BABABA
└── c
├── d
│ └── dd
│ └── ddd
└── e
]
> rt
%NlRbudHl:1% {} [
├── a
│ └── BABABA
│ └── c
│ ├── d [...]
│ └── e
└── X
└── Y -> /a/BABABA/c/e
]
>
*/
/*
nv-file-vfs-local-alone# nv_vfs_lcl_show -w TEST/tree/ -id 3
[ 4, 0, 2, 0, 4, 1, 'BABABA', {}, undefined, Set(0) {} ]
nv-file-vfs-local-alone#
*/ await rt.rename("a/BABABA/c","C2");
/*
> await rt.rename("a/BABABA/c","C2");
C2 %NlRbudHl:4% {
"meta": 6666,
"data": "abcdefg"
} [
C2
├── d
│ └── dd
│ └── ddd
└── e
]
> rt
%NlRbudHl:1% {} [
├── a
│ └── BABABA
│ └── C2
│ ├── d [...]
│ └── e
└── X
└── Y -> /a/BABABA/C2/e
]
>
*/
/*
nv-file-vfs-local-alone# tree TEST/tree/
TEST/tree/
└── ___disk___
├── ___data___
│ ├── 1
│ ├── 10
│ ├── 2
│ ├── 3
│ ├── 4
│ ├── 5
│ ├── 6
│ ├── 7
│ ├── 8
│ └── 9
└── ___meta___.json
2 directories, 11 files
nv-file-vfs-local-alone#
*/
swap
var X = rt.cd("X");
await X.swap("/a/BABABA/C2");
/*
> X
X %NlRbudHl:9% {} [
X
└── Y -> /a/BABABA/C2/e
]
> await X.swap("/a/BABABA/C2");
C2 %NlRbudHl:4% {
"meta": 6666,
"data": "abcdefg"
} [
C2
├── d
│ └── dd
│ └── ddd
└── e
]
>
*/
/*
> X.$parent_
BABABA %NlRbudHl:3% {} [
BABABA
└── X
└── Y -> /C2/e
]
> c.$parent_
%NlRbudHl:1% {} [
├── a
│ └── BABABA
│ └── X
│ └── Y -> /C2/e
└── C2
├── d
│ └── dd
│ └── ddd
└── e
]
>
*/
/*
nv-file-vfs-local-alone# nv_vfs_lcl_show -w TEST/tree/ -id 4
[
5,
0,
1,
2,
8,
1,
'C2',
{ meta: 6666, data: 'abcdefg' },
undefined,
Set(0) {}
]
nv-file-vfs-local-alone#
nv-file-vfs-local-alone# nv_vfs_lcl_show -w TEST/tree/ -id 9
[ 10, 0, 3, 0, 10, 1, 'X', {}, undefined, Set(0) {} ]
nv-file-vfs-local-alone#
*/mv
await rt.mv("C2/e","R/S/T");
/*
> await rt.mv("C2/e","R/S/T");
T %NlRbudHl:8% {}
> rt
%NlRbudHl:1% {} [
├── a
│ └── BABABA
│ └── X
│ └── Y -> /R/S/T
├── C2
│ └── d
│ └── dd
│ └── ddd
└── R
└── S
└── T
]
> e
T %NlRbudHl:8% {}
>
*/
/*
nv-file-vfs-local-alone# nv_vfs_lcl_show -w TEST/tree/ -id 13
[ 14, 0, 1, 4, 14, 1, 'R', {}, undefined, Set(0) {} ]
nv-file-vfs-local-alone# nv_vfs_lcl_show -w TEST/tree/ -id 14
[ 8, 0, 13, 0, 8, 1, 'S', {}, undefined, Set(0) {} ]
nv-file-vfs-local-alone#
nv-file-vfs-local-alone# nv_vfs_lcl_show -w TEST/tree/ -id 8
[ 0, 0, 14, 0, 0, 2, 'T', {}, undefined, Set(1) { 10 } ]
nv-file-vfs-local-alone#
*/
/*
nv-file-vfs-local-alone# tree TEST/tree/
TEST/tree/
└── ___disk___
├── ___data___
│ ├── 1
│ ├── 10
│ ├── 13 //R
│ ├── 14 //S
│ ├── 2
│ ├── 3
│ ├── 4
│ ├── 5
│ ├── 6
│ ├── 7
│ ├── 8 //e -> T
│ └── 9
└── ___meta___.json
2 directories, 13 files
*/cp cp | cp -r
await rt.cp("/a","/C2");
/*
> await rt.cp("/a","/C2");
a %NlRbudHl:17% {} [
a
└── BABABA
└── X
└── Y -> /R/S/T
]
> rt
%NlRbudHl:1% {} [
├── a
│ └── BABABA
│ └── X
│ └── Y -> /R/S/T
├── C2
│ ├── d
│ │ └── dd
│ │ └── ddd
│ └── a //new a/
│ └── BABABA
│ └── X [...]
└── R
└── S
└── T
]
>
*/
/*
nv-file-vfs-local-alone# nv_vfs_lcl_show -w TEST/tree/ -id 17
[ 18, 0, 4, 5, 18, 1, 'a', {}, undefined, Set(0) {} ]
nv-file-vfs-local-alone# nv_vfs_lcl_show -w TEST/tree/ -id 18
[
19, 0,
17, 0,
19, 1,
'BABABA', {},
undefined, Set(0) {}
]
nv-file-vfs-local-alone# nv_vfs_lcl_show -w TEST/tree/ -id 19
[ 20, 0, 18, 0, 20, 1, 'X', {}, undefined, Set(0) {} ]
nv-file-vfs-local-alone#
nv-file-vfs-local-alone# nv_vfs_lcl_show -w TEST/tree/ -id 20
[ 0, 0, 19, 0, 0, 3, 'Y', {}, 8, Set(0) {} ]
nv-file-vfs-local-alone#
nv-file-vfs-local-alone# nv_vfs_lcl_show -w TEST/tree/ -id 8
[ 0, 0, 14, 0, 0, 2, 'T', {}, undefined, Set(2) { 10, 20 } ] //T is refed by two nodes(old and copied)
nv-file-vfs-local-alone#
*/
/*
nv-file-vfs-local-alone# tree TEST/tree/
TEST/tree/
└── ___disk___
├── ___data___
│ ├── 1
│ ├── 10
│ ├── 13
│ ├── 14
│ ├── 17 // new a/
│ ├── 18 // new BABABA/
│ ├── 19 // new X/
│ ├── 2
│ ├── 20 // new Y -> /R/S/T
│ ├── 3
│ ├── 4
│ ├── 5
│ ├── 6
│ ├── 7
│ ├── 8
│ └── 9
└── ___meta___.json
2 directories, 17 files
*/
/*
> var new_Y = rt.cd("C2/a/BABABA/X/Y")
> new_Y
Y -> /R/S/T %NlRbudHl:20% {}
> new_Y.$id_
20
> var old_Y = rt.cd("a/BABABA/X/Y")
> old_Y
Y -> /R/S/T %NlRbudHl:10% {}
>
> new_Y !== old_Y
true
> new_Y.ref_ === old_Y.ref_
true
>
var T = new_Y.ref_
> T.refed_by_
Set(2) { Y -> /R/S/T %NlRbudHl:10% {}, Y -> /R/S/T %NlRbudHl:20% {} }
>
*/ await rt.cp("/a","E"); //E not-exist AND not-end-with(/) SO its a no-dir fnode nvpath
/*
> rt
%NlRbudHl:1% {} [
├── a
│ └── BABABA
│ └── X
│ └── Y -> /R/S/T
├── C2
│ ├── d
│ │ └── dd
│ │ └── ddd
│ └── a
│ └── BABABA
│ └── X [...]
├── R
│ └── S
│ └── T
└── E
└── BABABA
└── X
└── Y -> /R/S/T
]
>
*/
/*
nv-file-vfs-local-alone# tree TEST/
TEST/
├── tree
│ └── ___disk___
│ ├── ___data___
│ │ ├── 1
│ │ ├── 10
│ │ ├── 13
│ │ ├── 14
│ │ ├── 17
│ │ ├── 18
│ │ ├── 19
│ │ ├── 2
│ │ ├── 20
│ │ ├── 22 // E/
│ │ ├── 23 // E/BABABA/
│ │ ├── 24 // E/BABABA/X/
│ │ ├── 25 // E/BABABA/X/Y
│ │ ├── 3
│ │ ├── 4
│ │ ├── 5
│ │ ├── 6
│ │ ├── 7
│ │ ├── 8
│ │ └── 9
│ └── ___meta___.json
└── tst.js
3 directories, 22 files
*/
/*
> T.refed_by_
Set(3) {
Y -> /R/S/T %NlRbudHl:10% {},
Y -> /R/S/T %NlRbudHl:20% {},
Y -> /R/S/T %NlRbudHl:25% {}
}
>
*/remove (put into RECYCLE)
await rt.rm("E");
/*
> await rt.rm("E");
Y -> <empty> %NlRbudHl:25% {}
> T.refed_by_
Set(2) { Y -> /R/S/T %NlRbudHl:10% {}, Y -> /R/S/T %NlRbudHl:20% {} }
>
> rt
%NlRbudHl:1% {} [
├── a
│ └── BABABA
│ └── X
│ └── Y -> /R/S/T
├── C2
│ ├── d
│ │ └── dd
│ │ └── ddd
│ └── a
│ └── BABABA
│ └── X [...]
└── R
└── S
└── T
]
// only removed from root, but still in disk
> disk.fdisk()
[
%NlRbudHl:1% {} [
├── a
│ └── BABABA
│ └── X
│ └── Y -> /R/S/T
├── C2
│ ├── d
│ │ └── dd
│ │ └── ddd
│ └── a
│ └── BABABA
│ └── X [...]
└── R
└── S
└── T
],
E %NlRbudHl:22% {} [ //-----can RECYCLE
E
└── BABABA
└── X
└── Y -> <empty>
]
]
>
nv-file-vfs-local-alone# tree TEST/
TEST/
├── tree
│ └── ___disk___
│ ├── ___data___
│ │ ├── 1
│ │ ├── 10
│ │ ├── 13
│ │ ├── 14
│ │ ├── 17
│ │ ├── 18
│ │ ├── 19
│ │ ├── 2
│ │ ├── 20
│ │ ├── 22
│ │ ├── 23
│ │ ├── 24
│ │ ├── 25
│ │ ├── 3
│ │ ├── 4
│ │ ├── 5
│ │ ├── 6
│ │ ├── 7
│ │ ├── 8
│ │ └── 9
│ └── ___meta___.json
└── tst.js
3 directories, 22 files
*/
/*
nv-file-vfs-local-alone# nv_vfs_lcl_show -w TEST/tree/ -id 22
[ 23, 0, 0/*pr===0 means root OR removed*/, 0, 23, 1, 'E', {}, undefined, Set(0) {} ]
nv-file-vfs-local-alone#
*/ln to noexist path
await rt.ln("R/S/T/f","R/S/m");
/*
> await rt.ln("R/S/T/f","R/S/m");
Uncaught 'link_to_dst_not_exist'
>
*/erase (REAL destroy)
await rt.touch("R/S/m/");
/*
> await rt.touch("R/S/m/");
m{} %NlRbudHl:27% {}
> rt
%NlRbudHl:1% {} [
├── a
│ └── BABABA
│ └── X
│ └── Y -> /R/S/T
├── C2
│ ├── d
│ │ └── dd
│ │ └── ddd
│ └── a
│ └── BABABA
│ └── X [...]
└── R
└── S
├── T
└── m{}
]
>
*/
/*
nv-file-vfs-local-alone# tree TEST/
TEST/
├── tree
│ └── ___disk___
│ ├── ___data___
│ │ ├── 1
│ │ ├── 10
│ │ ├── 13
│ │ ├── 14
│ │ ├── 17
│ │ ├── 18
│ │ ├── 19
│ │ ├── 2
│ │ ├── 20
│ │ ├── 22
│ │ ├── 23
│ │ ├── 24
│ │ ├── 25
│ │ ├── 27 // R/S/m/
│ │ ├── 3
│ │ ├── 4
│ │ ├── 5
│ │ ├── 6
│ │ ├── 7
│ │ ├── 8
│ │ └── 9
│ └── ___meta___.json
└── tst.js
3 directories, 23 files
nv-file-vfs-local-alone# nv_vfs_lcl_show -w TEST/tree/ -id 27
[ 0, 0, 14, 8, 0, 1, 'm', {}, undefined, Set(0) {} ]
nv-file-vfs-local-alone#
*/
/*
> rt.cd("R/S").$id_
14
>
nv-file-vfs-local-alone# nv_vfs_lcl_show -w TEST/tree/ -id 14
[ 8, 0, 13, 0, 27, 1, 'S', {}, undefined, Set(0) {} ]
nv-file-vfs-local-alone#
*/
await rt.erase("R/S");
/*
> await rt.erase("R/S");
%erased% {}
> disk.get_nd_with_id(14)
null
>
>
> rt
%qdavcEiJ:1% {} [
├── a
│ └── BABABA
│ └── X
│ └── Y -> <empty>
├── C2
│ ├── d
│ │ └── dd
│ │ └── ddd
│ └── a
│ └── BABABA
│ └── X [...]
└── R{}
]
> disk.fdisk()
[
%qdavcEiJ:1% {} [
├── a
│ └── BABABA
│ └── X
│ └── Y -> <empty>
├── C2
│ ├── d
│ │ └── dd
│ │ └── ddd
│ └── a
│ └── BABABA
│ └── X [...]
└── R{}
],
E %qdavcEiJ:22% {} [
E
└── BABABA
└── X
└── Y -> <empty>
]
]
>
*/
/*
nv-file-vfs-local-alone# tree TEST/
TEST/
├── tree
│ └── ___disk___
│ ├── ___data___
│ │ ├── 1
│ │ ├── 10
│ │ ├── 13
│ │ ├── 17
│ │ ├── 18
│ │ ├── 19
│ │ ├── 2
│ │ ├── 20
│ │ ├── 22
│ │ ├── 23
│ │ ├── 24
│ │ ├── 25
│ │ ├── 3
│ │ ├── 4
│ │ ├── 5
│ │ ├── 6
│ │ ├── 7
│ │ └── 9
│ └── ___meta___.json
└── tst.js
3 directories, 20 files
*/Disk
disk
disk.fdisk()
disk.fdisk()[0] === rt
/*
disk.defrag disk.erase_isolated disk.gen_entry disk.ids_
disk.init_rent_pool disk.isolates_ disk.load_nd_from_dump disk.load_nd_from_nest
disk.merge disk.node disk.nodes_ disk.rent
disk.rent_pool_ disk.rtrn disk.slot disk.slots_
disk.tree disk.trees_
disk.dump disk.fdisk disk.get_nd_with_id disk.invalid_slinks_
disk.removed_
disk.btype disk.compressed disk.creat_root_dir
disk.init disk.is_inited disk.set_btype_to_json disk.set_btype_to_v8
disk.set_compressed disk.unset_compressed disk.workdir_
disk.fc disk.fid disk.idpool disk.lb
disk.lc disk.max_size disk.pr disk.rb
*/load
/*
>.exit //alone mode NOT support multi disk r/w sync
// master /slaves mode in another pkg: nv-file-vfs-local-master AND nv-file-vfs-local-cluster
node
*/
const {
Disk,FileNode,
load_disk_from_single_file_dump,
load_disk_from_workdir,
load_disk_from_real_dir, //only for test
} = require("nv-file-vfs-local-alone");slow
var disk2 = await x.load_disk_from_workdir("./TEST/tree")
disk2
/*
> disk2
Disk {
fid: 'NlRbudHlLHMHQUjtJsAfGkKDdsBjffmYiegc',
max_size: 20000,
idpool: { minid_: 1, maxid_: 20000, used_: 18, lefted_: 19982 }
}
> disk2.fdisk()
[
%qdavcEiJ:1% {} [
├── a
│ └── BABABA
│ └── X
│ └── Y -> <empty>
├── C2
│ ├── d
│ │ └── dd
│ │ └── ddd
│ └── a
│ └── BABABA
│ └── X [...]
└── R{}
],
E %qdavcEiJ:22% {} [
E
└── BABABA
└── X
└── Y -> <empty>
]
]
> var rt = disk2.fdisk()[0]
> rt
%qdavcEiJ:1% {} [
├── a
│ └── BABABA
│ └── X
│ └── Y -> <empty>
├── C2
│ ├── d
│ │ └── dd
│ │ └── ddd
│ └── a
│ └── BABABA
│ └── X [...]
└── R{}
]
>
*/
creat fake disk of a real dir ,very slow
- load from real-dir will NOT load the content,
- ONLY construct the dir/file/soft-link relations
- invalid soft-link (linked-to NOT exist) will be IGNORED
- external soft-link (linked-to NOT belong to real-dir) will be IGNORED
- char-dev/blk-dev/fifo/sock (such as dev/tty,temperary eventfd,...) will be IGNORED (performance reason)
- load from real-dir ONLY for test, its loading very SLOW if files more than 3000
load_disk_from_real_dir : {async:true} (
realdir: LinuxPathStr,
workdir: LinuxPathStr,
max_size: Int {>=0 AND <=2**20 AND <dflt IS 200000>}, --max fnode size of vdisk
//-----------------------------------------------------------------------------------------
load_data: Boolean {dflt IS false}, --dont use this(perf bad)
load_func: (data,self)=> {/*..*/} {<dflt IS _load.DFLT_LOAD_FUNC>} --dont use this(perf bad)
):Disk
var disk = await load_disk_from_real_dir("./","./TEST/curr")
// creat a fake disk in "./TEST/curr" for real dir "./"
var rt = disk.fdisk()[0]
> rt
%jCFldmUP:1% {} [
├── .npmignore
├── README.md
├── TEST
│ ├── tree
│ │ └── ___disk___
│ │ ├── ___data___ [...]
│ │ └── ___meta___.json
│ ├── tst.js
│ ├── tst2.js
│ └── tst3.js
├── index.js
├── node_modules
│ ├── .bin
│ │ └── nanoid -> /node_modules/nanoid/bin/nanoid.cjs
│ ├── .package-lock.json
│ ├── charcodes
│ │ ├── README.md
│ │ ├── lib
│ │ │ ├── index.js
│ │ │ ├── index.js.flow
│ │ │ ├── index.mjs
│ │ │ └── index.mjs.flow
│ │ ├── package.json
│ │ └── src
│ │ └── index.js
...........
...........
... 182 more items
]
> disk.nodes_.filter(nd=>nd.refed_by_.size)
[ nanoid.cjs %jCFldmUP:274% {} ]
>
> disk.nodes_.filter(nd=>nd.refed_by_.size)[0]
nanoid.cjs %jCFldmUP:274% {}
> var nd = disk.nodes_.filter(nd=>nd.refed_by_.size)[0]
> nd.refed_by_
Set(1) {
nanoid -> /node_modules/nanoid/bin/nanoid.cjs %jCFldmUP:300% {}
}
>load_disk_from_single_file_dump fast
var etc;
var p = (async () => {
console.log(new Date);
etc = await load_disk_from_real_dir("/etc","../etc",5000);
console.log(new Date);
})();
> etc
Disk {
fid: 'pkEKxqnsHhRrjXgXZJuweqIafZITXzhokaiq',
max_size: 5000,
idpool: { minid_: 1, maxid_: 5000, used_: 2022, lefted_: 2978 }
}
> etc.fdisk()
[
%pkEKxqns:1% {} [
├── .java
│ └── .systemPrefs
│ ├── .system.lock
│ └── .systemRootModFile
├── .pwd.lock
├── NetworkManager
│ ├── NetworkManager.conf
│ ├── conf.d
│ │ └── default-wifi-powersave-on.conf
.....
var dumped = etc.dump()
/*
> dumped
{
fid: 'bQlZlGCAQuEspwWjlfuFQrXlwQhFaPzdRjvx',
maxid: 5000,
struct: {
ids: [
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12,
.....
97, 98, 99, 100,
... 1922 more items
],
fc: [
2, 227, 0, 228, 234, 236, 237, 0, 0, 0, 251, 252,
253, 0, 263, 265, 279, 281, 0, 288, 0, 0, 291, 0,
0, 293, 296, 298, 0, 0, 299, 300, 303, 307, 346, 347,
......
*/
> await etc.dump_to_single_file("./etc.single")
true
>
/*
root@dev2:/opt/JS/nvfile-tst# ls -l | egrep etc
-rw-r--r-- 1 root root 16139 Jun 2 13:16 etc.single
*/
.exit
root@dev2:/opt/JS/nvfile-tst# mkdir etc2
const {load_disk_from_single_file_dump} = require("nv-file-vfs-local-alone");
var etc2 = await load_disk_from_single_file_dump("./etc.single","./etc2")
> var etc2 = await load_disk_from_single_file_dump("./etc.single","./etc2")
--- when using load_disk_from_single_file_dump, during the INIT-STAGE
--- disk has a is_readonly() method
--- its for waiting sync-with-local-fs
--- when it is TRUE, write/mv/cp/rename.... is forbidden
--- after sync-with-local-fs finished, it will be FALSE
> etc2
Disk {
fid: 'bQlZlGCAQuEspwWjlfuFQrXlwQhFaPzdRjvx',
max_size: 5000,
idpool: { minid_: 1, maxid_: 5000, used_: 2022, lefted_: 2978 }
}
> etc2.fdisk()
[
%bQlZlGCA:1% {} [
├── .java
│ └── .systemPrefs
│ ├── .system.lock
│ └── .systemRootModFile
├── .pwd.lock
├── NetworkManager
│ ├── NetworkManager.conf
│ ├── conf.d
│ │ └── default-wifi-powersave-on.conf
│ ├── dispatcher.d
│ │ ├── 01-ifupdown
....TEST
root@dev2:/opt/JS# umount /mnt/nas //this dir too large umount it
root@dev2:/opt/JS# ls -l /mntuse very slow load from real dir:
less than 500
root@dev2:/opt/JS# tree /boot | wc -l
311
var boot;
var p = (async () => {
console.log(new Date);
boot = await load_disk_from_real_dir("/boot","../boot",5000);
console.log(new Date);
})();
/*
2022-06-02T06:58:02.331Z
2022-06-02T06:58:05.736Z
> boot.fdisk()
[
%QzxfEeeh:1% {} [
├── System.map-5.4.0-110-generic
├── System.map-5.4.0-113-generic
├── config-5.4.0-110-generic
├── config-5.4.0-113-generic
├── efi
│ └── EFI
│ ├── BOOT
│ │ ├── BOOTX64.EFI
│ │ ├── fbx64.efi
│ │ └── mmx64.efi
│ └── ubuntu
│ ├── BOOTX64.CSV
......
*/less than 1000
root@dev:/opt/JS# tree /sbin | wc -l
510
const {load_disk_from_real_dir} = require("nv-file-vfs-local-alone");
var sbin;
var p = (async () => {
console.log(new Date);
sbin = await load_disk_from_real_dir("/usr/sbin","../sbin",5000);
console.log(new Date);
})();
2022-06-02T08:28:31.078Z
2022-06-02T08:28:37.002Z
> sbin.fdisk()
[
%kqTjRyGe:1% {} [
├── ModemManager
├── NetworkManager
├── aa-remove-unknown
├── aa-status
├── aa-teardown
├── accessdb
├── add-shell
├── addgnupghome
├── adduser
├── agetty
├── apache2
├── apparmor_parser
├── apparmor_status -> /aa-status
├── applygnupgdefaults
├── aptd
..... less than 2000
root@dev2:/opt/JS/nvfile-tst# tree /bin/ | wc -l
1381
> var bin;
>
> var p = (async () => {
... console.log(new Date);
... sbin = await load_disk_from_real_dir("/usr/bin","../bin",5000);
... console.log(new Date);
... })();
2022-06-02T08:44:16.477Z
2022-06-02T08:44:31.400Z
> bin =sbin
Disk {
fid: 'CTiRhZaLtmhTpHSMyPfkjQhfeiarxSESbdCm',
max_size: 5000,
idpool: { minid_: 1, maxid_: 5000, used_: 1298, lefted_: 3702 }
}
> bin.fdisk()
[
%CTiRhZaL:1% {} [
├── VGAuthService
├── Xephyr
├── Xorg
├── Xwayland
├── [
├── aa-enabled
├── aa-exec
├── aclocal-1.16
├── add-apt-repositoryless than 20000
root@dev2:/opt/JS# tree /var | wc -l
14749
> var _var;
>
> var p = (async () => {
... console.log(new Date);
... _var = await load_disk_from_real_dir("/var","../_var",20000);
... console.log(new Date);
... })();
2022-06-02T08:51:44.645Z
2022-06-02T08:54:16.429Z
> _var.fdisk()
[
%hfTUUpwx:1% {} [
├── backups
│ ├── alternatives.tar.0
│ ├── alternatives.tar.1.gz
│ ├── alternatives.tar.2.gz
│ ├── alternatives.tar.3.gz
│ ├── alternatives.tar.4.gz
│ ├── alternatives.tar.5.gz
│ ├── alternatives.tar.6.gz
│ ├── apt.extended_states.0
│ ├── apt.extended_states.1.gz
│ ├── apt.extended_states.2.gz
│ ├── apt.extended_states.3.gz
│ ├── apt.extended_states.4.gz
│ ├── apt.extended_states.5.gz
│ ├── apt.extended_states.6.gz
.....less than 150000
root@dev2:/opt/JS# tree /proc | wc -l
128805
const {load_disk_from_real_dir} = require("nv-file-vfs-local-alone");
var proc;
var p = (async () => {
console.log(new Date);
proc = await load_disk_from_real_dir("/proc","../proc",300000);
console.log(new Date);
})();
/*
2022-06-02T09:17:17.635Z
2022-06-02T09:30:50.890Z
> proc.fdisk()
[
%ruFcCbiq:1% {} [
├── 1
│ ├── arch_status
│ ├── attr
│ │ ├── apparmor
│ │ │ ├── current
│ │ │ ├── exec
│ │ │ └── prev
│ │ ├── current
│ │ ├── display
│ │ ├── exec
│ │ ├── fscreate
│ │ ├── keycreate
│ │ ├── prev
│ │ ├── smack
....
*/less than 500000
root@dev2:/opt/JS# tree /home | wc -l
411068
const {load_disk_from_real_dir} = require("nv-file-vfs-local-alone");
var home;
var p = (async () => {
console.log(new Date);
home = await load_disk_from_real_dir("/home","../home",500000);
console.log(new Date);
})();
/*
2022-06-02T09:35:34.864Z
-------------------------------------very slow 1 hour
2022-06-02T10:40:51.672Z
> home.fdisk()
[
%gpMFeDjk:1% {} [
├── WWW
│ ├── SEDITER
│ │ ├── css
│ │ │ ├── dark.css
│ │ │ └── light.css
│ │ ├── filelist.php
│ │ ├── fileop.php
│ │ ├── font
│ │ │ ├── Unnamed-Regular.vfc
│ │ │ ├── myfont{}
│ │ │ ├── myfont.ttf
│ │ │ └── tu.ai
│ │ ├── jquery.js
│ │ ├── js
│ │ │ ├── cookie.js
│ │ │ ├── fileclient.js
│ │ │ ├── gbk.js
│ │ │ ├── lkeditor.js
│ │ │ └── vs [...]
│ │ ├── login.php
│ │ ├── logo_red.png
│ │ ├── logon.php
│ │ ├── upload.php
│ │ ├── uploadclient.php
│ │ ├── user.php
*/whole linux
var fake_linux;
(async () =>{
console.log(new Date);
fake_linux = await load_disk_from_real_dir("/","../linux",2000000);
//my ubuntu has about 80_00000 inodes
console.log(new Date);
})();
//------very very long time... about 2-3 hours using slow load disk from workdir
//now all fnodes in memory
//reload from workdir for test
/*
>exit
node
>
*/
var fake_linux;
(async () =>{
console.log(new Date);
fake_linux = await load_disk_from_workdir("../linux");
console.log(new Date);
})();
//------ long time... 1463098 nodes need 2 hours...
/*
2022-06-04T16:37:02.218Z
Promise {
<pending>,
[Symbol(async_id_symbol)]: 66,
[Symbol(trigger_async_id_symbol)]: 5,
[Symbol(destroyed)]: { destroyed: false }
}
>
> 2022-06-04T18:40:10.628Z
*/
/*
> fake_linux
Disk {
fid: 'IqbYFsVaigYqSgeAcSDksojjjVPUXrLWnrAs',
max_size: 2000000,
idpool: { minid_: 1, maxid_: 2000000, used_: 1463098, lefted_: 536902 }
}
> fake_linux.fdisk()
[
%IqbYFsVa:1% {} [
├── boot
│ ├── System.map-5.4.0-110-generic
│ ├── System.map-5.4.0-113-generic
│ ├── config-5.4.0-110-generic
│ ├── config-5.4.0-113-generic
│ ├── efi
│ │ └── EFI
│ │ ├── BOOT [...]
│ │ └── ubuntu [...]
│ ├── grub
│ │ ├── fonts
│ │ │ └── unicode.pf2
│ │ ├── grub.cfg
│ │ ├── grubenv
│ │ ├── unicode.pf2
│ │ └── x86_64-efi
.....
*/dump
(async () =>{
console.log(new Date);
await fake_linux.dump_to_single_file("./linux.single");
console.log(new Date);
})();
/*
2022-06-05T03:25:07.263Z
.... ---- about 5 minutes to dump,1463098 paths(no content)
2022-06-05T03:29:57.234Z
*/
/*
root@dev2:/opt/JS/nvfile-tst# ls -l
total 5436
-rw-r--r-- 1 root root 5511021 Jun 5 03:29 linux.single
---- need about 5M space to save 1463098 paths
*/using load_disk_from_single_file_dump
/*
>.exit
node
*/
var fake_linux;
(async ()=> {
console.log(new Date);
fake_linux = await load_disk_from_single_file_dump("./linux.single","./linux");
console.log(new Date);
})();
/*
2022-06-05T05:35:14.313Z
....... //about 25s to load_disk_from_single_file_dump
2022-06-05T05:35:36.642Z
*/
> fake_linux.is_readonly() //only-read and search is supported now
true
>
root@dev2:/opt/JS/nvfile-tst# tree -af linux | wc -l
78123
root@dev2:/opt/JS/nvfile-tst# tree -af linux | wc -l
91107
root@dev2:/opt/JS/nvfile-tst# tree -af linux | wc -l
92158
root@dev2:/opt/JS/nvfile-tst# tree -af linux | wc -l
93477
root@dev2:/opt/JS/nvfile-tst# tree -af linux | wc -l
94502
....###search performance
//write to random node
var nd0 = fake_linux.nodes_[Math.floor(Math.random()*600000)]
await nd0.awrite({___data:new Map([[11,22],[33,44]])})
> nd0.read()
{ ___data: Map(2) { 11 => 22, 33 => 44 } }
>
>
var nd1 = fake_linux.nodes_[Math.floor(Math.random()*600000)]
await nd1.awrite({___data:new Map([[110,220],[330,440]])})
> nd1.read()
{ ___data: Map(2) { 110 => 220, 330 => 440 } }
>
var rt = fake_linux.fdisk()[0];
var tst = require("nv-facutil-simple-test")
var sdfs = rt.$sdfs_;
function find_fst() {
for(let nd of sdfs) {
let data = nd.___data;
if(data?.get(11)===22) {
return(nd)
}
}
}
> tst.sync(100,find_fst)
{ rounds: 100, f: [Function: find_fst], costed: 1812.2085847854614 }
> tst.sync(100,find_fst)
{ rounds: 100, f: [Function: find_fst], costed: 1818.3960609436035 }
> tst.sync(100,find_fst)
{ rounds: 100, f: [Function: find_fst], costed: 1811.705943107605 }
> tst.sync(100,find_fst)
{ rounds: 100, f: [Function: find_fst], costed: 1821.3589589595795 }
>
>
> var arr = fake_linux.nodes_
> arr.length
1463098
>
function find_fst_on_arr() {
for(let nd of arr) {
let data = nd.___data;
if(data?.get(11)===22) {
return(nd)
}
}
}
> tst.sync(100,find_fst_on_arr)
{
rounds: 100,
f: [Function: find_fst_on_arr],
costed: 619.2632658481598
}
> tst.sync(100,find_fst_on_arr)
{
rounds: 100,
f: [Function: find_fst_on_arr],
costed: 620.0158529281616
}
> tst.sync(100,find_fst_on_arr)
{
rounds: 100,
f: [Function: find_fst_on_arr],
costed: 612.9748179912567
}
> tst.sync(1000,find_fst_on_arr)
{
rounds: 1000,
f: [Function: find_fst_on_arr],
costed: 5919.528462171555
}
>
> var sys = rt.cd("sys")
> sys
sys %IqbYFsVa:17% {} [
sys
├── block{}
├── bus
│ ├── ac97
│ │ ├── devices{}
│ │ ├── drivers{}
│ │ ├── drivers_autoprobe
│ │ ├── drivers_probe
│ │ └── uevent
......
function find_childs() {
let rslt = []
let g = sys.$gen_sdfs_next();
for(let nd of g) {
let pnd = nd.$parent_;
if(pnd.$_raw_tag_ === "ac97" && nd.$_raw_tag_.includes("dri") ) {
rslt.push(nd)
}
}
return(rslt)
}
/*
> find_childs()
[
drivers{} %IqbYFsVa:37450% {},
drivers_autoprobe %IqbYFsVa:37451% {},
drivers_probe %IqbYFsVa:37452% {}
]
>
> tst.sync(10,find_childs)
{ rounds: 10, f: [Function: find_childs], costed: 1095.4665169715881 }
>
*/
> process.memoryUsage()
{
rss: 772853760,
heapTotal: 563757056,
heapUsed: 536161288,
external: 1234037,
arrayBuffers: 25012
}
>
read AND search test
var slinks = rt.$sdfs_.filter(nd=>nd.is_slink())
> slinks[0]
initrd.img.old -> /boot/initrd.img-5.4.0-110-generic %IqbYFsVa:29% {}
>
var linktos = rt.$sdfs_.filter(nd=>nd.refed_by_.size>0)
/*
> linktos
[
initrd.img-5.4.0-110-generic %IqbYFsVa:27% {},
initrd.img-5.4.0-113-generic %IqbYFsVa:28% {},
vmlinuz-5.4.0-110-generic %IqbYFsVa:30% {},
vmlinuz-5.4.0-113-generic %IqbYFsVa:31% {},
shm %IqbYFsVa:47% {} [
shm
└── multipath
├── failed_wwids{}
└── find_multipaths{}
],
.....
> linktos[0].refed_by_
Set(1) {
initrd.img.old -> /boot/initrd.img-5.4.0-110-generic %IqbYFsVa:29% {}
}
>
*/
/*
> tst.sync(10,()=>fake_linux.nodes_.filter(nd=>nd.is_slink()))
{ rounds: 10, f: [Function (anonymous)], costed: 2679.3424689769745 }
>
tst.sync(1000,()=>rt.ls());
> tst.sync(100000,()=>rt.ls())
{
rounds: 100000,
f: [Function (anonymous)],
costed: 624.6834530830383
}
>
*/
/*
var paths = fake_linux.nodes_.map(r=>r.nvpath_)
> paths[0]
'/'
> paths[1]
'/boot/'
> paths[2]
'/cdrom/'
> paths[3]
'/dev/'
> paths[4]
'/etc/'
> paths[400]
'/proc/217233/'
> paths[4000]
'/proc/168921/pagemap'
>
> tst.sync(1,()=>fake_linux.nodes_.map(r=>r.nvpath_));
{ rounds: 1, f: [Function (anonymous)], costed: 3764.210354089737 }
>
*/CONCLUSION
1. for project/lib files
normally, a js/ts project/lib dir will have no more than 3000 files(including node\_modules)
, persistence with local fs is OK2. for html/css
normally, a html/css web will have no more than 20_0000 files(most web page < 10000),
, persistence with local fs is just so so 3. for js runtime
normally, for a medium application,
it will keep around 200_0000+ objects(if NOT considering GC), if simulate GC with local fs,
the write frequence will be very HIGH, persistence with local fs is a BAD idea, need persistence with
a good dababase OR redis-like mem-database
METHODS
###FileNode
self define read/awrite methods, using add_read_method add_awrite_method
rt.read rt.aread
rt.write rt.awrite
tag getter
rt.$_raw_tag_ //real name
rt.$tag_ //for display using
rt.nvpath_ //path string
rt.ref_ // soft link node point to -> ref_actions
rt.cd(path)
rt.mkdir(path) //path always be treated as dir
rt.touch(path) // if path ends with "/" same as mkdir ,
rt.ln(linkto_path,linkfrom_path) //
rt.rm(path) // rm + rm -r ; removed nodes can be found using disk.removed_;
rt.erase(path) // rm + rm -r ; erased nodes will disappear
rt.rename(path,name)
rt.rename_self(name) //rename self
rt.swap(p0,p1)
rt.mv(srcp,dstp)node type
rt.is_dir()
rt.is_file()
rt.is_slink()internal init using
rt.is_undef()
rt.set_as_dir()
rt.set_as_file()
rt.set_as_slink()relations
other methods begin with $, provide ways to find sibling/parent/.... disk
disk.max_size //max_nodes when <= 1000000 will be fast.
// can only be set when new Disk(max_size)
// max_size cant change after new Disk
disk.creat_root_dir()
disk.fdisk() // list all "root-dir"
disk.removed_ // nodes be removed BUT not erased APIS
new Disk(
max_size,
workdir="./",
dtype='v8', // v8 OR json , default is v8
compress=true, //use brotli to compress , default true
)
//default 10000 , if more than 1000000 , performance BAD
await disk.dump_to_single_file(single_dump_path); //fast
await load_disk_from_single_file_dump(single_dump_path,workdir); //fast
await load_disk_from_workdir(workdir); //very-slow
await load_disk_from_real_dir(realdir,workdir); //very-very-slowLICENSE
- ISC