Add drivers section to Data.toml for autoloading (#20)

This allows modules which provide data storage drivers to be
automatically loaded when DataSets itself is loaded, providing a
declarative data environment workflow while bypassing world age
issues we'd get from loading these on demand.

It seems somewhat unclear whether interfering with code loading
like this is a good idea, but I think only time and some experience
with this mechanism will tell...

Author: c42f

src/DataSets.jl

@@ -4,6 +4,7 @@ using UUIDs
 using TOML
 using SHA
 using ResourceContexts
+using Base: PkgId
 
 export DataSet, dataset, @datafunc, @datarun
 export Blob, BlobTree, newfile, newdir
@@ -215,6 +216,8 @@ identifier, `nothing` is returned.
 """
 project_name(data_project::AbstractDataProject) = nothing
 
+data_drivers(proj::AbstractDataProject) = []
+
 #-------------------------------------------------------------------------------
 """
     DataProject
@@ -224,11 +227,15 @@ Names are unique within the project.
 """
 struct DataProject <: AbstractDataProject
     datasets::Dict{String,DataSet}
+    drivers::Vector{Dict{String,Any}}
 end
 
-DataProject() = DataProject(Dict{String,DataSet}())
+DataProject() = DataProject(Dict{String,DataSet}(), Vector{Dict{String,Any}}())
+
+DataProject(project::AbstractDataProject) = DataProject(Dict(pairs(project)),
+                                                        Vector{Dict{String,Any}}())
 
-DataProject(project::AbstractDataProject) = DataProject(Dict(pairs(project)))
+data_drivers(project::DataProject) = project.drivers
 
 function _fill_template(toml_path, toml_str)
     # Super hacky templating for paths relative to the toml file.
@@ -276,6 +283,14 @@ function load_project(config::AbstractDict; kws...)
         dataset = DataSet(dataset_conf)
         link_dataset(proj, dataset.name => dataset)
     end
+    if haskey(config, "drivers")
+        _check_keys(config, DataProject, ["drivers"=>AbstractVector])
+        for driver_conf in config["drivers"]
+            _check_keys(driver_conf, DataProject, ["type"=>String, "name"=>String, "module"=>Dict])
+            _check_keys(driver_conf["module"], DataProject, ["name"=>String, "uuid"=>String])
+            push!(proj.drivers, driver_conf)
+        end
+    end
     proj
 end
 
@@ -363,6 +378,8 @@ end
 
 StackedDataProject() = StackedDataProject([])
 
+data_drivers(stack::StackedDataProject) = vcat(data_drivers.(stack.projects)...)
+
 function Base.keys(stack::StackedDataProject)
     names = []
     for project in stack.projects
@@ -479,8 +496,23 @@ PROJECT = StackedDataProject()
 # deprecated. TODO: Remove dependency on this from JuliaHub
 _current_project = DataProject()
 
+_isprecompiling() = ccall(:jl_generating_output, Cint, ()) == 1
+
 function __init__()
-    global PROJECT = create_project_stack(ENV)
+    # Triggering Base.require for storage drivers during precompilation should
+    # be unnecessary and can cause problems if those driver modules use
+    # Requires-like code loading.
+    if !_isprecompiling()
+        global PROJECT = create_project_stack(ENV)
+        for proj in PROJECT.projects
+            try
+                add_storage_driver(proj)
+            catch exc
+                @error "Could not load storage drivers from data project" #=
+                    =# project=proj exception=(exc,catch_backtrace())
+            end
+        end
+    end
 end
 
 dataset(name) = dataset(PROJECT, name)
@@ -494,6 +526,7 @@ May be renamed in a future version.
 """
 function load_project!(path_or_config)
     new_project = load_project(path_or_config, auto_update=true)
+    add_storage_driver(new_project)
     pushfirst!(PROJECT, new_project)
     # deprecated: _current_project reflects only the initial version of the
     # project on *top* of the stack.
@@ -527,6 +560,33 @@ function add_storage_driver((name,opener)::Pair)
     end
 end
 
+function add_storage_driver(project::AbstractDataProject)
+    for conf in data_drivers(project)
+        if conf["type"] != "storage"
+            # Anticipate there might be layer drivers too
+            continue
+        end
+        pkgid = PkgId(UUID(conf["module"]["uuid"]), conf["module"]["name"])
+        if Base.haskey(Base.package_locks, pkgid)
+            # Hack: Avoid triggering another call to require() for packages
+            # which are already in the process of being loaded. (This would
+            # result in a deadlock!)
+            #
+            # Obviously this depends on Base internals...
+            continue
+        end
+        mod = Base.require(pkgid)
+        driver_name = conf["name"]
+        # Module itself does add_storage_driver() inside its __init__
+        # TODO: Is this a good workflow?
+        lock(_storage_drivers_lock) do
+            get(_storage_drivers, driver_name) do
+                error("Package $pkgid did not provide storage driver $driver_name")
+            end
+        end
+    end
+end
+
 function _find_driver(dataset)
     storage_config = dataset.storage
     driver_name = get(storage_config, "driver") do

src/file_data_projects.jl

@@ -118,6 +118,8 @@ end
 
 Base.pairs(proj::AbstractTomlFileDataProject) = pairs(_get_cached(proj))
 
+data_drivers(proj::AbstractTomlFileDataProject) = data_drivers(_get_cached(proj))
+
 #-------------------------------------------------------------------------------
 """
 Data project which automatically updates based on a TOML file on the local

test/DriverAutoloadData.toml

@@ -0,0 +1,21 @@
+data_config_version = 0
+
+[[datasets]]
+description="Test dynamic loading of drivers"
+name="dummy_storage_blob"
+uuid="785b3cdc-428e-426f-a3f7-3f6ae88a9637"
+
+    [datasets.storage]
+    driver="DummyTomlStorage"
+    type="Blob"
+    data="data_from_dummy_backend"
+
+#-------------------------------------------------------------------------------
+
+[[drivers]]
+type="storage"
+name="DummyTomlStorage"
+
+    [drivers.module]
+    name="DummyStorageBackends"
+    uuid="89b7a33a-382e-4698-a931-421b088d35a2"

test/driver_autoload.jl

@@ -0,0 +1,9 @@
+@testset "Automatic code loading for drivers" begin
+    empty!(DataSets.PROJECT)
+    pushfirst!(LOAD_PATH, abspath("drivers"))
+    ENV["JULIA_DATASETS_PATH"] = joinpath(@__DIR__, "DriverAutoloadData.toml")
+    DataSets.__init__()
+    @test haskey(DataSets._storage_drivers, "DummyTomlStorage")
+
+    @test open(String, dataset("dummy_storage_blob")) == "data_from_dummy_backend"
+end

test/drivers/DummyStorageBackends/Project.toml

@@ -0,0 +1,5 @@
+name="DummyStorageBackends"
+uuid="89b7a33a-382e-4698-a931-421b088d35a2"
+
+[deps]
+DataSets = "c9661210-8a83-48f0-b833-72e62abce419"

test/drivers/DummyStorageBackends/src/DummyStorageBackends.jl

@@ -0,0 +1,23 @@
+module DummyStorageBackends
+
+using DataSets
+
+struct DummyBackend
+    data
+end
+
+function Base.open(f::Function, ::Type{IO}, storage::DummyBackend, path; kws...) where {T}
+    @assert isempty(path)
+    f(IOBuffer(storage.data))
+end
+
+function connect_dummy_backend(f, config, ds)
+    storage = DummyBackend(config["data"])
+    f(Blob(storage))
+end
+
+function __init__()
+    DataSets.add_storage_driver("DummyTomlStorage"=>connect_dummy_backend)
+end
+
+end

test/runtests.jl

@@ -125,3 +125,4 @@ include("entrypoint.jl")
 include("repl.jl")
 include("DataTomlStorage.jl")
 include("backend_compat.jl")
+include("driver_autoload.jl")