From a54c286a7546df7e22f3efdb072f379bfeedcb95 Mon Sep 17 00:00:00 2001 From: "mahlon@martini.nu" Date: Sun, 14 Feb 2021 09:47:04 +0000 Subject: [PATCH] Start blocking out some documentation. - Fix some C rdoc so it is parsed correctly. - Fill out transaction testing. - Populate docs for DB options. FossilOrigin-Name: f54dbfacf2dda100a116fdcc856ca5231e249f23238ca9d4355618e3a380a8f8 --- .pryrc | 3 +- README.md | 118 ++++++++++++++++++++++++++++++++++++- ext/mdbx_ext/database.c | 44 ++++++++------ ext/mdbx_ext/mdbx_ext.c | 10 +++- ext/mdbx_ext/mdbx_ext.h | 6 +- gem.deps.rb | 1 + lib/mdbx.rb | 4 -- lib/mdbx/database.rb | 84 +++++++++++++++++++++++--- spec/lib/helper.rb | 13 +++- spec/mdbx/database_spec.rb | 103 ++++++++++++++++++++++++++++++-- spec/mdbx/stats_spec.rb | 2 +- 11 files changed, 345 insertions(+), 43 deletions(-) diff --git a/.pryrc b/.pryrc index 062583e..4a3bcfc 100644 --- a/.pryrc +++ b/.pryrc @@ -11,5 +11,6 @@ rescue Exception => e e.backtrace.join( "\n\t" ) end -db = MDBX::Database.open( 'tmp/testdb', max_collections: 5 ) +# db = MDBX::Database.open( 'tmp/testdb' ) + diff --git a/README.md b/README.md index 0498482..2b7da8f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -# Ruby MDBX + +# Ruby::MDBX home : https://code.martini.nu/ruby-mdbx @@ -99,7 +100,7 @@ development. ## License -Copyright (c) 2020, Mahlon E. Smith +Copyright (c) 2020-2021 Mahlon E. Smith All rights reserved. Redistribution and use in source and binary forms, with or without @@ -127,3 +128,116 @@ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +Ruby MDBX +========= + +https://erthink.github.io/libmdbx/intro.html + +Notes on the libmdbx environment for ruby: + + - A **database** is contained in a file, normally wrapped in directory for it's associated lock. + - Each database can contain multiple named **collections**. + - Each collection can contain any number of **keys**, and their associated **values**. A collection may optionally support multiple values per key (or duplicate keys, which is the same thing). + - A **cursor** lets you iterate a collection's keys and values in order. + (Note, this should be enumerable and built in to the Ruby interface) + - A **snapshot** is a self-consistent read-only view of the database. It stays the same even if some other thread or process makes changes. *The only way to access keys and values is within a snapshot*. + - A **transaction** is a writable snapshot. Changes made within a transaction are private until committed. *The only way to modify the database is within a transaction*. + + +Example Usage +---------------- + +### Create database handle + +```ruby +db = MDBX::Database.create( "/path/to/file", options ) +db = MDBX::Database.open( "/path/to/file", options ) + +# perhaps a block mode that yields the handle, closing on block exit? +MDBX::Database.open( 'database' ) do |db| + puts db[ 'key1' ] +end +``` + +### Access data + +```ruby +db[ 'key1' ] #=> val +# In the backend, automatically creates the snapshot, retrieves the value, and removes the snapshot before returning. + +# read-only block +db.snapshot do + db[ 'key1' ] #=> val + ... +end +# This is much faster for retrieving many values + +# Maybe have a snapshot object that acts like a DB while it exists? +snap = db.snapshot +snap[ 'whatever' ] #=> data +snap.close +``` + +### Write data + +```ruby +db[ 'key1' ] = val +# In the backend, automatically creates a transaction, stores the value, and closes the transaction before returning. + +# writable block +db.transaction do + db[ 'key1' ] = val +end +# Much faster for writing many values, should commit on success or abort on any exception + +# Maybe have a transaction object that acts like a DB while it exists? +# ALL OTHER TRANSACTIONS will block until this is closed +txn = db.transaction +txn[ 'whatever' ] = data +txn.commit # or txn.abort +``` + +### Collections + +Identical interface to top-level databases. Just have to pull the collection first. + +```ruby +collection = db.collection( 'stuff' ) # raise if nonexistent +# This now works just like the main db object + +collection.transaction.do + ... +end +``` + +### Cleaning up + +```ruby +db.close +``` + + + +### Stats! + + +TODO +------- + + gem install mdbx -- --with-opt-dir=/usr/local + + - [ ] Multiple value per key -- .insert, .delete? iterator for multi-val keys + - [ ] each_pair? + - [ ] document how serialization works + - [ ] document everything, really + - [x] transaction/snapshot blocks + - [ ] Arbitrary keys instead of forcing to strings? + - [ ] Disallow collection switching if there is an open transaction + + + + + diff --git a/ext/mdbx_ext/database.c b/ext/mdbx_ext/database.c index 68fe559..56cab5f 100644 --- a/ext/mdbx_ext/database.c +++ b/ext/mdbx_ext/database.c @@ -69,7 +69,7 @@ rmdbx_free( void *db ) { if ( db ) { rmdbx_close_all( db ); - free( db ); + xfree( db ); } } @@ -88,7 +88,7 @@ rmdbx_close( VALUE self ) /* * call-seq: - * db.closed? #=> false + * db.closed? => false * * Predicate: return true if the database environment is closed. */ @@ -102,7 +102,7 @@ rmdbx_closed_p( VALUE self ) /* * call-seq: - * db.in_transaction? #=> false + * db.in_transaction? => false * * Predicate: return true if a transaction (or snapshot) * is currently open. @@ -117,6 +117,7 @@ rmdbx_in_transaction_p( VALUE self ) /* * Open the DB environment handle. + * */ VALUE rmdbx_open_env( VALUE self ) @@ -139,7 +140,7 @@ rmdbx_open_env( VALUE self ) mdbx_env_set_maxreaders( db->env, db->settings.max_readers ); /* Set an upper boundary (in bytes) for the database map size. */ - if ( db->settings.max_size > -1 ) + if ( db->settings.max_size ) mdbx_env_set_geometry( db->env, -1, -1, db->settings.max_size, -1, -1, -1 ); rc = mdbx_env_open( db->env, db->path, db->settings.env_flags, db->settings.mode ); @@ -251,7 +252,9 @@ rmdbx_rb_closetxn( VALUE self, VALUE write ) * call-seq: * db.clear * - * Empty the database (or collection) on disk. Unrecoverable! + * Empty the current collection on disk. If collections are not enabled + * or the database handle is set to the top-level (main) db - this + * deletes *all data* on disk. Fair warning, this is not recoverable! */ VALUE rmdbx_clear( VALUE self ) @@ -313,7 +316,7 @@ rmdbx_val_for( VALUE self, VALUE arg ) /* call-seq: - * db.keys #=> [ 'key1', 'key2', ... ] + * db.keys => [ 'key1', 'key2', ... ] * * Return an array of all keys in the current collection. */ @@ -351,7 +354,7 @@ rmdbx_keys( VALUE self ) /* call-seq: - * db[ 'key' ] #=> value + * db[ 'key' ] => value * * Convenience method: return a single value for +key+ immediately. */ @@ -390,7 +393,7 @@ rmdbx_get_val( VALUE self, VALUE key ) /* call-seq: - * db[ 'key' ] = value #=> value + * db[ 'key' ] = value * * Convenience method: set a single value for +key+ */ @@ -432,7 +435,7 @@ rmdbx_put_val( VALUE self, VALUE key, VALUE val ) /* * call-seq: - * db.statistics #=> (hash of stats) + * db.statistics => (hash of stats) * * Returns a hash populated with various metadata for the opened * database. @@ -449,17 +452,19 @@ rmdbx_stats( VALUE self ) /* - * Gets or sets the sub-database "collection" that read/write operations apply to. + * call-seq: + * db.collection( 'collection_name' ) => db + * db.collection( nil ) => db (main) + * + * Gets or sets the sub-database "collection" that read/write + * operations apply to. * Passing +nil+ sets the database to the main, top-level namespace. * If a block is passed, the collection automatically reverts to the * prior collection when it exits. * - * db.collection( 'collection_name' ) # => db - * db.collection( nil ) # => db (main) - * * db.collection( 'collection_name' ) do * [ ... ] - * end #=> reverts to the previous collection name + * end => reverts to the previous collection name * */ VALUE @@ -475,8 +480,13 @@ rmdbx_set_subdb( int argc, VALUE *argv, VALUE self ) return rb_str_new_cstr( db->subdb ); } + /* Provide a friendlier error message if max_collections is 0. */ + if ( db->settings.max_collections == 0 ) + rb_raise( rmdbx_eDatabaseError, "Unable to change collection: collections are not enabled." ); + /* All transactions must be closed when switching database handles. */ - if ( db->txn ) rb_raise( rmdbx_eDatabaseError, "Unable to change collection: finish current transaction" ); + if ( db->txn ) + rb_raise( rmdbx_eDatabaseError, "Unable to change collection: transaction open" ); /* Retain the prior database collection if a * block was passed. */ @@ -505,7 +515,7 @@ rmdbx_set_subdb( int argc, VALUE *argv, VALUE self ) db->subdb = prev_db; rmdbx_close_dbi( db ); } - free( prev_db ); + xfree( prev_db ); } return self; @@ -554,7 +564,7 @@ rmdbx_database_initialize( int argc, VALUE *argv, VALUE self ) db->settings.mode = 0644; db->settings.max_collections = 0; db->settings.max_readers = 0; - db->settings.max_size = -1; + db->settings.max_size = 0; /* Options setup, overrides. */ diff --git a/ext/mdbx_ext/mdbx_ext.c b/ext/mdbx_ext/mdbx_ext.c index 3bea4c2..9836b51 100644 --- a/ext/mdbx_ext/mdbx_ext.c +++ b/ext/mdbx_ext/mdbx_ext.c @@ -14,12 +14,18 @@ Init_mdbx_ext() { rmdbx_mMDBX = rb_define_module( "MDBX" ); - /* The backend library version. */ VALUE version = rb_str_new_cstr( mdbx_version.git.describe ); + /* The backend MDBX library version. */ rb_define_const( rmdbx_mMDBX, "LIBRARY_VERSION", version ); + /* A generic exception class for internal Database errors. */ rmdbx_eDatabaseError = rb_define_class_under( rmdbx_mMDBX, "DatabaseError", rb_eRuntimeError ); - rmdbx_eRollback = rb_define_class_under( rmdbx_mMDBX, "Rollback", rb_eRuntimeError ); + + /* + * Raising an MDBX::Rollback exception from within a transaction + * discards all changes and closes the transaction. + */ + rmdbx_eRollback = rb_define_class_under( rmdbx_mMDBX, "Rollback", rb_eRuntimeError ); rmdbx_init_database(); } diff --git a/ext/mdbx_ext/mdbx_ext.h b/ext/mdbx_ext/mdbx_ext.h index 8b2ba84..a3f45bb 100644 --- a/ext/mdbx_ext/mdbx_ext.h +++ b/ext/mdbx_ext/mdbx_ext.h @@ -4,8 +4,8 @@ #include "mdbx.h" -#ifndef MDBX_EXT_0_9_2 -#define MDBX_EXT_0_9_2 +#ifndef MDBX_EXT_0_9_3 +#define MDBX_EXT_0_9_3 #define RMDBX_TXN_ROLLBACK 0 #define RMDBX_TXN_COMMIT 1 @@ -63,5 +63,5 @@ extern void rmdbx_close_txn( rmdbx_db_t*, int ); extern VALUE rmdbx_gather_stats( rmdbx_db_t* ); -#endif /* define MDBX_EXT_0_9_2 */ +#endif /* define MDBX_EXT_0_9_3 */ diff --git a/gem.deps.rb b/gem.deps.rb index abe4b1c..0865dc5 100644 --- a/gem.deps.rb +++ b/gem.deps.rb @@ -9,5 +9,6 @@ group( :development ) do gem 'rspec', '~> 3.9' gem 'rubocop', '~> 0.93' gem 'simplecov', '~> 0.12' + gem 'simplecov-console', '~> 0.9' end diff --git a/lib/mdbx.rb b/lib/mdbx.rb index 092c966..020d703 100644 --- a/lib/mdbx.rb +++ b/lib/mdbx.rb @@ -10,10 +10,6 @@ require 'mdbx_ext' module MDBX # The version of this gem. - # - # Note: the MDBX library version this gem was built - # against can be found in the 'LIBRARY_VERSION' constant. - # VERSION = '0.0.1' end # module MDBX diff --git a/lib/mdbx/database.rb b/lib/mdbx/database.rb index 3f7d0b9..6604171 100644 --- a/lib/mdbx/database.rb +++ b/lib/mdbx/database.rb @@ -5,19 +5,89 @@ require 'mdbx' unless defined?( MDBX ) -# TODO: rdoc +# The primary class for interacting with an MDBX database. # class MDBX::Database - ### Open an existing (or create a new) mdbx database at filesystem - ### +path+. In block form, the database is automatically closed. + ### call-seq: + ### MDBX::Database.open( path ) => db + ### MDBX::Database.open( path, options ) => db + ### + ### Open an existing (or create a new) mdbx database at filesystem + ### +path+. In block form, the database is automatically closed + ### when the block exits. ### - ### MDBX::Database.open( path ) -> db - ### MDBX::Database.open( path, options ) -> db ### MDBX::Database.open( path, options ) do |db| - ### db[ 'key' ] #=> value + ### db[ 'key' ] = value ### end ### + ### Passing options modify various database behaviors. See the libmdbx + ### documentation for detailed information. + ### + ### ==== Options + ### + ### [:mode] + ### Whe creating a new database, set permissions to this 4 digit + ### octal number. Defaults to `0644`. Set to `0` to never automatically + ### create a new file, only opening existing databases. + ### + ### [:max_collections] + ### Set the maximum number of "subdatabase" collections allowed. By + ### default, collection support is disabled. + ### + ### [:max_readers] + ### Set the maximum number of allocated simultaneous reader slots. + ### + ### [:max_size] + ### Set an upper boundary (in bytes) for the database map size. + ### The default is 10485760 bytes. + ### + ### [:nosubdir] + ### When creating a new database, don't put the data and lock file + ### under a dedicated subdirectory. + ### + ### [:readonly] + ### Reject any write attempts while using this database handle. + ### + ### [:exclusive] + ### Access is restricted to this process handle. Other attempts + ### to use this database (even in readonly mode) are denied. + ### + ### [:compat] + ### Avoid incompatibility errors when opening an in-use database with + ### unknown or mismatched flag values. + ### + ### [:writemap] + ### Trade safety for speed for databases that fit within available + ### memory. (See MDBX documentation for details.) + ### + ### [:no_threadlocal] + ### Parallelize read-only transactions across threads. Writes are + ### always thread local. (See MDBX documentatoin for details.) + ### + ### [:no_readahead] + ### Disable all use of OS readahead. Potentially useful for + ### random reads wunder low memory conditions. Default behavior + ### is to dynamically choose when to use or omit readahead. + ### + ### [:no_memory_init] + ### Skip initializing malloc'ed memory to zeroes before writing. + ### + ### [:coalesce] + ### Attempt to coalesce items for the garbage collector, + ### potentialy increasing the chance of unallocating storage + ### earlier. + ### + ### [:lifo_reclaim] + ### Recycle garbage collected items via LIFO, instead of FIFO. + ### Depending on underlying hardware (disk write-back cache), this + ### could increase write performance. + ### + ### [:no_metasync] + ### A system crash may sacrifice the last commit for a potentially + ### large write performance increase. Database integrity is + ### maintained. + ### def self::open( *args, &block ) db = new( *args ) @@ -40,7 +110,7 @@ class MDBX::Database private_class_method :new - # The options used to instantiate this database. + # Options used when instantiating this database handle. attr_reader :options # The path on disk of the database. diff --git a/spec/lib/helper.rb b/spec/lib/helper.rb index f5bfff2..f5198d2 100644 --- a/spec/lib/helper.rb +++ b/spec/lib/helper.rb @@ -5,7 +5,18 @@ if ENV[ 'COVERAGE' ] require 'simplecov' - SimpleCov.start + require 'simplecov-console' + SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new([ + SimpleCov::Formatter::HTMLFormatter, + SimpleCov::Formatter::Console, + ]) + SimpleCov.start do + add_filter 'spec' + # enable_coverage :branch + add_group "Needing tests" do |file| + file.covered_percent < 90 + end + end end require 'pathname' diff --git a/spec/mdbx/database_spec.rb b/spec/mdbx/database_spec.rb index 90a6706..4093243 100644 --- a/spec/mdbx/database_spec.rb +++ b/spec/mdbx/database_spec.rb @@ -6,6 +6,12 @@ require_relative '../lib/helper' RSpec.describe( MDBX::Database ) do + after( :all ) do + db = described_class.open( TEST_DATABASE.to_s ) + db.clear + db.close + end + it "disallows direct calls to #new" do expect{ described_class.new }. to raise_exception( NoMethodError, /private/ ) @@ -60,8 +66,6 @@ RSpec.describe( MDBX::Database ) do db[ 'test' ] = nil expect( db['test'] ).to be_nil - - expect { db[ 'test' ] = nil }.to_not raise_exception end it "can return an array of its keys" do @@ -84,9 +88,9 @@ RSpec.describe( MDBX::Database ) do it "fail if the max_collections option is not specified when opening" do db.close db = described_class.open( TEST_DATABASE.to_s ) - db.collection( 'bucket' ) - - expect{ db['key'] = true }.to raise_exception( /MDBX_DBS_FULL/ ) + expect{ + db.collection( 'bucket' ) + } .to raise_exception( /not enabled/ ) end it "disallows regular key/val storage for namespace keys" do @@ -130,5 +134,94 @@ RSpec.describe( MDBX::Database ) do expect( db['bucket'] ).to be_nil end end + + + context 'transactions' do + + let!( :db ) { described_class.open( TEST_DATABASE.to_s, max_collections: 5 ) } + + after( :each ) do + db.close + end + + it "knows when a transaction is currently open" do + expect( db.in_transaction? ).to be_falsey + db.snapshot + expect( db.in_transaction? ).to be_truthy + db.abort + expect( db.in_transaction? ).to be_falsey + + db.snapshot do + expect( db.in_transaction? ).to be_truthy + end + expect( db.in_transaction? ).to be_falsey + end + + it "throws an error if changing collection mid-transaction" do + db.snapshot do + expect{ db.collection('nope') }. + to raise_exception( MDBX::DatabaseError, /transaction open/ ) + end + end + + it "are re-entrant" do + 3.times { db.snapshot } + expect( db.in_transaction? ).to be_truthy + end + + it "refuse to allow writes for read-only snapshots" do + db.snapshot + expect{ db[1] = true }. + to raise_exception( MDBX::DatabaseError, /permission denied/i ) + end + + it "revert changes via explicit rollbacks" do + db[ 1 ] = true + db.transaction + db[ 1 ] = false + expect( db[ 1 ] ).to be_falsey + db.rollback + expect( db[ 1 ] ).to be_truthy + end + + it "revert changes via uncaught exceptions" do + db[ 1 ] = true + expect { + db.transaction do + db[ 1 ] = false + raise "boom" + end + }.to raise_exception( RuntimeError, "boom" ) + expect( db[ 1 ] ).to be_truthy + end + + it "revert changes via explicit exceptions" do + db[ 1 ] = true + expect { + db.transaction do + db[ 1 ] = false + raise MDBX::Rollback, "boom!" + end + }.to_not raise_exception + expect( db[ 1 ] ).to be_truthy + end + + it "write changes after commit" do + db[ 1 ] = true + db.transaction + db[ 1 ] = false + expect( db[ 1 ] ).to be_falsey + db.commit + expect( db[ 1 ] ).to be_falsey + end + + it "automatically write changes after block" do + db[ 1 ] = true + db.transaction do + db[ 1 ] = false + end + expect( db[ 1 ] ).to be_falsey + end + end end diff --git a/spec/mdbx/stats_spec.rb b/spec/mdbx/stats_spec.rb index 4d45520..f0f9635 100644 --- a/spec/mdbx/stats_spec.rb +++ b/spec/mdbx/stats_spec.rb @@ -4,7 +4,7 @@ require_relative '../lib/helper' -RSpec.fdescribe( MDBX::Database ) do +RSpec.describe( MDBX::Database ) do let!( :db ) { described_class.open( TEST_DATABASE.to_s, max_readers: 500 ) }