Storage driver for data embedded in Data.toml (#15)

* Storage driver for data embedded in Data.toml

* Blobs are stored as base64 encoded strings
* Trees are stored as dictionaries

While we're at it, start to clean up storage backend interface.

Also deprecate `abspath(::RelPath)` depending on `pwd()`, as the
assumption that this refers to the filesystem, and that depending on the
global `pwd()` is actually reasonable seem pretty dubious in hindsight.

Author: c42f

Project.toml

@@ -5,18 +5,19 @@ version = "0.2.3"
 
 [deps]
 AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
-ResourceContexts = "8d208092-d35c-4dd3-a0d7-8325f9cce6b4"
+Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 ReplMaker = "b873ce64-0db9-51f5-a568-4457d8e49576"
+ResourceContexts = "8d208092-d35c-4dd3-a0d7-8325f9cce6b4"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]
 AbstractTrees = "0.3"
 ReplMaker = "0.2"
-TOML = "1"
 ResourceContexts = "0.1"
+TOML = "1"
 julia = "1.5"
 
 [extras]

src/BlobTree.jl

@@ -300,6 +300,8 @@ Base.abspath(tree::BlobTree) = AbsPath(tree.root, tree.path)
 function Base.getindex(tree::BlobTree, path::RelPath)
     relpath = joinpath(tree.path, path)
     root = tree.root
+    # TODO: Make this more efficient by moving this work to the storage backend?
+    # Sort of like an equivalent of `stat`?
     if isdir(root, relpath)
         BlobTree(root, relpath)
     elseif isfile(root, relpath)
@@ -316,9 +318,9 @@ function Base.getindex(tree::BlobTree, name::AbstractString)
 end
 
 # We've got a weird mishmash of path vs tree handling here.
-# TODO: Can we refactor this to cleanly separate the filesystem commands (which
-# take abstract paths?) from BlobTree and Blob which act as an abstraction over
-# the filesystem or other storage mechanisms?
+# TODO: Can we refactor this to cleanly separate the filesystem-like commands
+# (which take abstract paths?) from BlobTree and Blob which act as an
+# abstraction over the filesystem or other storage mechanisms?
 function Base.joinpath(tree::BlobTree, r::RelPath)
     AbsPath(tree.root, joinpath(tree.path, r))
 end
@@ -335,6 +337,10 @@ function Base.readdir(tree::BlobTree)
     readdir(tree.root, tree.path)
 end
 
+function Base.keys(tree::BlobTree)
+    readdir(tree.root, tree.path)
+end
+
 function Base.rm(tree::BlobTree; kws...)
     rm(tree.root, tree.path; kws...)
 end

src/DataSets.jl

@@ -461,7 +461,7 @@ Additional projects may be added or removed from the stack with `pushfirst!`,
 """
 PROJECT = StackedDataProject()
 
-# deprecated.
+# deprecated. TODO: Remove dependency on this from JuliaHub
 _current_project = DataProject()
 
 function __init__()
@@ -562,6 +562,7 @@ include("BlobTree.jl")
 
 # Builtin backends
 include("filesystem.jl")
+include("DataTomlStorage.jl")
 
 # Backends
 # include("ZipTree.jl")

src/DataTomlStorage.jl

@@ -0,0 +1,127 @@
+using Base64
+
+"""
+Storage driver which keeps the data embedded within the TOML file itself.
+Useful for small amounts of self-contained data.
+
+## Metadata spec
+
+For Blob:
+```
+    [datasets.storage]
+    driver="TomlDataStorage"
+    type="Blob"
+    data=\$(base64encode(data))
+```
+
+For BlobTree:
+```
+    [datasets.storage]
+    driver="TomlDataStorage"
+    type="BlobTree"
+
+        [datasets.storage.data.\$(dirname1)]
+        "\$(filename1)" = \$(base64encode(data1))
+        "\$(filename2)" = \$(base64encode(data2))
+
+        [datasets.storage.data.\$(dirname2)]
+        ...
+```
+"""
+struct TomlDataStorage
+    dataset::DataSet
+    data::Union{String,Dict{String,Any}}
+end
+
+# Get TOML data at `path`, returning nothing if not present
+function _getpath(storage::TomlDataStorage, path::RelPath)
+    x = storage.data
+    for c in path.components
+        x = get(x, c, nothing)
+        !isnothing(x) || return nothing
+    end
+    x
+end
+
+#--------------------------------------------------
+# Storage data interface for trees
+
+Base.isdir(storage::TomlDataStorage, path::RelPath) = _getpath(storage, path) isa Dict
+Base.isfile(storage::TomlDataStorage, path::RelPath) = _getpath(storage, path) isa String
+Base.ispath(storage::TomlDataStorage, path::RelPath) = !isnothing(_getpath(storage, path))
+
+Base.summary(io::IO, storage::TomlDataStorage) = print(io, "Data.toml")
+
+function Base.readdir(storage::TomlDataStorage, path::RelPath)
+    try
+        tree = _getpath(storage, path)
+        !isnothing(tree) || KeyError(path)
+        sort!(collect(keys(tree::AbstractDict)))
+    catch
+        error("TOML storage requires trees to be as TOML dictionaries")
+    end
+end
+
+#--------------------------------------------------
+# Storage data interface for Blob
+
+function Base.open(func::Function, as_type::Type{IO},
+                   storage::TomlDataStorage, path; kws...)
+    @context func(@! open(as_type, storage, path; kws...))
+end
+
+@! function Base.open(::Type{Vector{UInt8}}, storage::TomlDataStorage, path;
+                      write=false, read=!write, kws...)
+    if write
+        error("Embedded data is read-only from within the DataSets interface")
+    end
+    try
+        str = _getpath(storage, path)
+        !isnothing(str) || KeyError(path)
+        base64decode(str::AbstractString)
+    catch
+        error("TOML storage requires data to be as base64 encoded strings")
+    end
+end
+
+@! function Base.open(::Type{IO}, storage::TomlDataStorage, path; kws...)
+    buf = @! open(Vector{UInt8}, storage, path; kws...)
+    IOBuffer(buf)
+end
+
+
+# TODO: The following should be factored out and implemented generically
+function Base.read(storage::TomlDataStorage, path::RelPath, ::Type{T}) where {T}
+    @context begin
+        io = @! open(IO, storage, path)
+        read(io, T)
+    end
+end
+
+function Base.read(storage::TomlDataStorage, path::RelPath)
+    @context @! open(Vector{UInt8}, storage, path)
+end
+
+
+#-------------------------------------------------------------------------------
+# Connect storage backend
+function connect_toml_data_storage(f, config, dataset)
+    type = config["type"]
+    data = get(config, "data", nothing)
+    if type == "Blob"
+        if !(data isa AbstractString)
+            error("TOML data storage requires string data in the \"storage.data\" key")
+        end
+        f(Blob(TomlDataStorage(dataset, data)))
+    elseif type == "BlobTree"
+        if !(data isa AbstractDict)
+            error("TOML data storage requires a dictionary in the \"storage.data\" key")
+        end
+        f(BlobTree(TomlDataStorage(dataset, data)))
+    else
+        throw(ArgumentError("DataSet type $type not supported for data embedded in Data.toml"))
+    end
+end
+
+add_storage_driver("TomlDataStorage"=>connect_toml_data_storage)
+

src/filesystem.jl

@@ -4,7 +4,7 @@
 #
 abstract type AbstractFileSystemRoot end
 
-# These underscore functions sys_abspath and sys_joinpath generate/joins OS-specific
+# These functions sys_abspath and sys_joinpath generate/joins OS-specific
 # _local filesystem paths_ out of logical paths. They should be defined only
 # for trees which are rooted in the actual filesystem.
 function sys_abspath(root::AbstractFileSystemRoot, path::RelPath)
@@ -17,30 +17,27 @@ sys_abspath(path::AbsPath) = sys_abspath(path.root, path.path)
 sys_abspath(tree::BlobTree) = sys_abspath(tree.root, tree.path)
 sys_abspath(file::Blob) = sys_abspath(file.root, file.path)
 
+#--------------------------------------------------
+# Storage data interface for trees
+#
+# TODO: Formalize this interface!
+
+## 1. Query
+
 # TODO: would it be better to express the following dispatch in terms of
 # AbsPath{<:AbstractFileSystemRoot} rather than usin double dispatch?
+
 Base.isdir(root::AbstractFileSystemRoot, path::RelPath) = isdir(sys_abspath(root, path))
 Base.isfile(root::AbstractFileSystemRoot, path::RelPath) = isfile(sys_abspath(root, path))
 Base.ispath(root::AbstractFileSystemRoot, path::RelPath) = ispath(sys_abspath(root, path))
-Base.read(root::AbstractFileSystemRoot, path::RelPath, ::Type{T}) where {T} =
-    read(sys_abspath(root, path), T)
-Base.read(root::AbstractFileSystemRoot, path::RelPath) where {T} =
-    read(sys_abspath(root, path))
 
 Base.summary(io::IO, root::AbstractFileSystemRoot) = print(io, sys_abspath(root))
 
-function Base.open(f::Function, as_type::Type{IO}, root::AbstractFileSystemRoot, path;
-                   kws...)
-    @context f(@! open(as_type, root, path; kws...))
-end
+Base.readdir(root::AbstractFileSystemRoot, path::RelPath) = readdir(sys_abspath(root, path))
 
-@! function Base.open(::Type{IO}, root::AbstractFileSystemRoot, path;
-                      write=false, read=!write, kws...)
-    if !iswriteable(root) && write
-        error("Error writing file at read-only path $path")
-    end
-    @! open(sys_abspath(root, path); read=read, write=write, kws...)
-end
+## 2. Mutation
+#
+# TODO: Likely requires rework!
 
 function Base.mkdir(root::AbstractFileSystemRoot, path::RelPath; kws...)
     if !iswriteable(root)
@@ -54,9 +51,49 @@ function Base.rm(root::AbstractFileSystemRoot, path::RelPath; kws...)
     rm(sys_abspath(root,path); kws...)
 end
 
-Base.readdir(root::AbstractFileSystemRoot, path::RelPath) = readdir(sys_abspath(root, path))
+#--------------------------------------------------
+# Storage data interface for Blob
+
+# TODO: Make this the generic implementation for AbstractDataStorage
+function Base.open(f::Function, as_type::Type{IO},
+                   root::AbstractFileSystemRoot, path; kws...)
+    @context f(@! open(as_type, root, path; kws...))
+end
+
+@! function Base.open(::Type{IO}, root::AbstractFileSystemRoot, path;
+                      write=false, read=!write, kws...)
+    if !iswriteable(root) && write
+        error("Error writing file at read-only path $path")
+    end
+    @! open(sys_abspath(root, path); read=read, write=write, kws...)
+end
+
+Base.read(root::AbstractFileSystemRoot, path::RelPath, ::Type{T}) where {T} =
+    read(sys_abspath(root, path), T)
+Base.read(root::AbstractFileSystemRoot, path::RelPath) =
+    read(sys_abspath(root, path))
 
 #--------------------------------------------------
+"""
+
+## Metadata spec
+
+For Blob:
+```
+    [datasets.storage]
+    driver="FileSystem"
+    type="Blob"
+    path=\$(path_to_file)
+```
+
+For BlobTree:
+```
+    [datasets.storage]
+    driver="FileSystem"
+    type="BlobTree"
+    path=\$(path_to_directory)
+```
+"""
 struct FileSystemRoot <: AbstractFileSystemRoot
     path::String
     read::Bool
@@ -72,12 +109,15 @@ iswriteable(root::FileSystemRoot) = root.write
 
 sys_abspath(root::FileSystemRoot) = root.path
 
-# For use outside DataSets, we will assume the special case that abspath() with
-# a RelPath refers to the current working directory on the local system.
-Base.abspath(relpath::RelPath) =
+function Base.abspath(relpath::RelPath)
+    Base.depwarn("""
+        `abspath(::RelPath)` defaults to using `pwd()` as the root of the path
+        but this leads to fragile code so will be removed in the future""",
+        :abspath)
     AbsPath(FileSystemRoot(pwd(); write=true, read=true), relpath)
+end
 
-#--------------------------------------------------
+#-------------------------------------------------------------------------------
 # Infrastructure for a somewhat more functional interface for creating file
 # trees than the fully mutable version we usually use.
 

test/Data.toml

@@ -23,7 +23,6 @@ uuid="b498f769-a7f6-4f67-8d74-40b770398f26"
     # type="text"
     # parameters={encoding="UTF-8"}
 
-
 #--------------------------------------------------
 [[datasets]]
 description="Gzipped CSV example"
@@ -55,3 +54,43 @@ uuid="e7fd7080-e346-4a68-9ca9-98593a99266a"
 
     # TODO: Add data maps here which expose it logically as a single CSV?
 
+
+#--------------------------------------------------
+[[datasets]]
+description="A data blob embedded in the TOML"
+name="embedded_blob"
+uuid="b498f769-a7f6-4f67-8d74-40b770398f26"
+
+    [datasets.storage]
+    driver="TomlDataStorage"
+    type="Blob"
+    data="AAAAAAAARUA="
+
+
+[[datasets]]
+description="A data tree embedded in the TOML"
+name="embedded_tree"
+uuid="b498f769-a7f6-4f67-8d74-40b770398f26"
+
+    [datasets.storage]
+    driver="TomlDataStorage"
+    type="BlobTree"
+
+# TOML.print(Dict("datasets"=>[Dict("storage"=>Dict("data"=>Dict(["d0$i"=>Dict(["$x.txt"=>base64encode("$i $x content") for x in ("a","b")]...) for i in 1:4]...)))]))
+
+        [datasets.storage.data.d01]
+        "b.txt" = "MSBiIGNvbnRlbnQ="
+        "a.txt" = "MSBhIGNvbnRlbnQ="
+
+        [datasets.storage.data.d02]
+        "b.txt" = "MiBiIGNvbnRlbnQ="
+        "a.txt" = "MiBhIGNvbnRlbnQ="
+
+        [datasets.storage.data.d03]
+        "b.txt" = "MyBiIGNvbnRlbnQ="
+        "a.txt" = "MyBhIGNvbnRlbnQ="
+
+        [datasets.storage.data.d04]
+        "b.txt" = "NCBiIGNvbnRlbnQ="
+        "a.txt" = "NCBhIGNvbnRlbnQ="
+