Multiple changes.

- Raise an error if invalid options are passed to the constructor.
- Bugfix: Ensure drop() only removes the specified collection.
- Fix initializer double memory allocation.
- Fix key/data object allocation: make garbage collection safe.
- Move common macros to the global header file.

FossilOrigin-Name: 98d3016bd25921dead39d9c5474712766b56519d575bc8cd960932b3fbc16b69

.hgignore

@@ -8,6 +8,7 @@ spec/\.status
 coverage/
 vendor/
 pkg/
+^tags$
 ^Session.vim$
 ^Makefile$
 

.pryrc

@@ -11,5 +11,6 @@ rescue Exception => e
 		e.backtrace.join( "\n\t" )
 end
 
-db = MDBX::Database.open( 'tmp/testdb', max_collections: 100 )
+db = MDBX::Database.open( 'tmp/testdb', max_collections: 50 )
+#db = MDBX::Database.open( 'tmp/testdb', duplicate_keys: true )
 

History.md

@@ -1,5 +1,14 @@
 # Release History for MDBX
 
+---
+## v0.3.1 [2021-05-16] Mahlon E. Smith <mahlon@martini.nu>
+
+Bugfix:
+
+ - #drop could potentially remove unintended data.  Yanked version
+   v0.3.0.
+
+
 ---
 ## v0.3.0 [2021-04-09] Mahlon E. Smith <mahlon@martini.nu>
 

README.md

@@ -340,13 +340,6 @@ information about the build environment, the database environment, and
 the currently connected clients.
 
 
-## TODO
-
-  - Expose more database/collection information to statistics
-  - Support libmdbx multiple values per key DUPSORT via `put`, `get`
-    Enumerators, and a 'value' argument for `delete`.
-
-
 ## Contributing
 
 You can check out the current development source with Mercurial via its

experiments/thread_usage.rb

@@ -37,7 +37,6 @@ def run_bench( db, msg )
 			threads.map( &:join )
 		end
 
-
 		# Long running transactions require a mutex across threads.
 		#
 		x.report( "txn per thread:" ) do
@@ -78,4 +77,3 @@ run_bench( db, "Default database flags:" )
 db = MDBX::Database.open( 'tmpdb', no_metasync: true )
 run_bench( db, "Disabled metasync:" )
 
-

ext/mdbx_ext/mdbx_ext.h

@@ -4,12 +4,23 @@
 
 #include "mdbx.h"
 
-#ifndef MDBX_EXT_0_9_3
+#ifndef RBMDBX_EXT
-#define MDBX_EXT_0_9_3
+#define RBMDBX_EXT
 
 #define RMDBX_TXN_ROLLBACK 0
 #define RMDBX_TXN_COMMIT 1
 
+/* Shortcut for fetching wrapped data structure.
+ */
+#define UNWRAP_DB( self, db ) \
+	rmdbx_db_t *db; \
+	TypedData_Get_Struct( self, rmdbx_db_t, &rmdbx_db_data, db )
+
+/* Raise if current DB is not open. */
+#define CHECK_HANDLE() \
+	if ( ! db->state.open ) rb_raise( rmdbx_eDatabaseError, "Closed database." )
+
+
 /*
  * A struct encapsulating an instance's DB
  * state and settings.
@@ -21,7 +32,8 @@ struct rmdbx_db {
 	MDBX_cursor *cursor;
 
     struct {
-       int env_flags;
+       unsigned int env_flags;
+       unsigned int db_flags;
        int mode;
        int open;
        int max_collections;
@@ -40,12 +52,11 @@ struct rmdbx_db {
 typedef struct rmdbx_db rmdbx_db_t;
 
 static const rb_data_type_t rmdbx_db_data;
-extern void rmdbx_free( void *db ); /* forward declaration for the allocator */
+
 
 /* ------------------------------------------------------------
  * Globals
  * ------------------------------------------------------------ */
-
 extern VALUE rmdbx_mMDBX;
 extern VALUE rmdbx_cDatabase;
 extern VALUE rmdbx_eDatabaseError;
@@ -55,13 +66,15 @@ extern VALUE rmdbx_eRollback;
 /* ------------------------------------------------------------
  * Functions
  * ------------------------------------------------------------ */
+extern void rmdbx_free( void *db ); /* forward declaration for the allocator */
 extern void Init_rmdbx ( void );
 extern void rmdbx_init_database ( void );
+extern void rmdbx_close_all( rmdbx_db_t* );
 extern void rmdbx_open_txn( rmdbx_db_t*, int );
 extern void rmdbx_close_txn( rmdbx_db_t*, int );
-
+extern void rmdbx_open_cursor( rmdbx_db_t* );
 extern VALUE rmdbx_gather_stats( rmdbx_db_t* );
 
 
-#endif /* define MDBX_EXT_0_9_3 */
+#endif /* define RBMDBX_EXT */
 

ext/mdbx_ext/stats.c

@@ -3,8 +3,6 @@
  * Expose a bunch of mdbx internals to ruby.
  * This is all largely stolen from mdbx_stat.c.
  *
- * Entry point is rmdbx_stats() in database.c.
- *
  */
 
 #include "mdbx_ext.h"
@@ -28,6 +26,32 @@ rmdbx_gather_build_stats( VALUE stat )
 }
 
 
+/*
+ * Grab current memory usage.  (Available since MDBX 0.10.0).
+ */
+void
+rmdbx_gather_memory_stats( VALUE stat )
+{
+	if (! ( MDBX_VERSION_MAJOR >= 0 && MDBX_VERSION_MINOR >= 10 ) )
+		return;
+
+	VALUE mem = rb_hash_new();
+	rb_hash_aset( stat, ID2SYM(rb_intern("system_memory")), mem );
+
+	intptr_t page_size;
+	intptr_t total_pages;
+	intptr_t avail_pages;
+
+	mdbx_get_sysraminfo( &page_size, &total_pages, &avail_pages );
+
+	rb_hash_aset( mem, ID2SYM(rb_intern("pagesize")),    LONG2FIX( page_size ) );
+	rb_hash_aset( mem, ID2SYM(rb_intern("total_pages")), LONG2FIX( total_pages ) );
+	rb_hash_aset( mem, ID2SYM(rb_intern("avail_pages")), LONG2FIX( avail_pages ) );
+
+	return;
+}
+
+
 /*
  * Metadata for the database file.
  */
@@ -80,6 +104,16 @@ rmdbx_gather_environment_stats(
 
 	rb_hash_aset( environ, ID2SYM(rb_intern("pagesize")),
 			INT2NUM(mstat.ms_psize) );
+	rb_hash_aset( environ, ID2SYM(rb_intern("branch_pages")),
+			LONG2NUM(mstat.ms_branch_pages) );
+	rb_hash_aset( environ, ID2SYM(rb_intern("leaf_pages")),
+			LONG2NUM(mstat.ms_leaf_pages) );
+	rb_hash_aset( environ, ID2SYM(rb_intern("overflow_pages")),
+			LONG2NUM(mstat.ms_overflow_pages) );
+	rb_hash_aset( environ, ID2SYM(rb_intern("btree_depth")),
+			INT2NUM(mstat.ms_depth) );
+	rb_hash_aset( environ, ID2SYM(rb_intern("entries")),
+			LONG2NUM(mstat.ms_entries) );
 	rb_hash_aset( environ, ID2SYM(rb_intern("last_txnid")),
 			INT2NUM(menvinfo.mi_recent_txnid) );
 	rb_hash_aset( environ, ID2SYM(rb_intern("last_reader_txnid")),
@@ -101,7 +135,7 @@ rmdbx_gather_environment_stats(
  *
  */
 int
-reader_list_callback(
+rmdbx_reader_list_cb(
 	void *ctx,
 	int num,
 	int slot,
@@ -148,7 +182,7 @@ rmdbx_gather_reader_stats(
 {
 	VALUE readers = rb_ary_new();
 
-	mdbx_reader_list( db->env, reader_list_callback, (void*)readers );
+	mdbx_reader_list( db->env, rmdbx_reader_list_cb, (void*)readers );
 	rb_hash_aset( stat, ID2SYM(rb_intern("readers")), readers );
 
 	return;
@@ -168,6 +202,7 @@ rmdbx_gather_stats( rmdbx_db_t *db )
 	MDBX_stat mstat;
 	MDBX_envinfo menvinfo;
 
+	rmdbx_gather_memory_stats( stat );
 	rmdbx_gather_build_stats( stat );
 
 	rmdbx_open_txn( db, MDBX_TXN_RDONLY );
@@ -183,9 +218,6 @@ rmdbx_gather_stats( rmdbx_db_t *db )
 	rmdbx_gather_environment_stats( stat, mstat, menvinfo );
 	rmdbx_gather_reader_stats( db, stat, mstat, menvinfo );
 
-	/* TODO: database and subdatabase stats */
-
 	return stat;
 }
 
-

lib/mdbx/database.rb

@@ -29,10 +29,23 @@ class MDBX::Database
 	### Unless otherwise mentioned, option keys are symbols, and values
 	### are boolean.
 	###
-	### [:mode]
+	### [:coalesce]
-	###   Whe creating a new database, set permissions to this 4 digit
+	###   Attempt to coalesce items for the garbage collector,
-	###   octal number.  Defaults to `0644`.  Set to `0` to never automatically
+	###   potentialy increasing the chance of unallocating storage
-	###   create a new file, only opening existing databases.
+	###   earlier.
+	###
+	### [:compatible]
+	###   Skip compatibility checks when opening an in-use database with
+	###   unknown or mismatched flag values.
+	###
+	### [:exclusive]
+	###   Access is restricted to the first opening process. Other attempts
+	###   to use this database (even in readonly mode) are denied.
+	###
+	### [:lifo_reclaim]
+	###   Recycle garbage collected items via LIFO, instead of FIFO.
+	###   Depending on underlying hardware (disk write-back cache), this
+	###   could increase write performance.
 	###
 	### [:max_collections]
 	###   Set the maximum number of "subdatabase" collections allowed. By
@@ -45,51 +58,38 @@ class MDBX::Database
 	###   Set an upper boundary (in bytes) for the database map size.
 	###   The default is 10485760 bytes.
 	###
-	### [:nosubdir]
+	### [:mode]
+	###   Whe creating a new database, set permissions to this 4 digit
+	###   octal number.  Defaults to `0644`.  Set to `0` to never automatically
+	###   create a new file, only opening existing databases.
+	###
+	### [:no_memory_init]
+	###   Skip initializing malloc'ed memory to zeroes before writing.
+	###
+	### [:no_metasync]
+	###   A system crash may sacrifice the last commit for a potentially
+	###   large write performance increase.  Database integrity is
+	###   maintained.
+	###
+	### [:no_subdir]
 	###   When creating a new database, don't put the data and lock file
 	###   under a dedicated subdirectory.
 	###
-	### [:readonly]
-	###   Reject any write attempts while using this database handle.
-	###
-	### [:exclusive]
-	###   Access is restricted to the first opening process. Other attempts
-	###   to use this database (even in readonly mode) are denied.
-	###
-	### [:compat]
-	###   Skip compatibility checks when opening an in-use database with
-	###   unknown or mismatched flag values.
-	###
-	### [:writemap]
-	###   Trade safety for speed for databases that fit within available
-	###   memory. (See MDBX documentation for details.)
-	###
-	### [:no_threadlocal]
-	###   Parallelize read-only transactions across threads.  Writes are
-	###   always thread local. (See MDBX documentatoin for details.)
-	###
 	### [:no_readahead]
 	###   Disable all use of OS readahead.  Potentially useful for
 	###   random reads wunder low memory conditions.  Default behavior
 	###   is to dynamically choose when to use or omit readahead.
 	###
-	### [:no_memory_init]
+	### [:no_threadlocal]
-	###   Skip initializing malloc'ed memory to zeroes before writing.
+	###   Parallelize read-only transactions across threads.  Writes are
+	###   always thread local. (See MDBX documentatoin for details.)
 	###
-	### [:coalesce]
+	### [:readonly]
-	###   Attempt to coalesce items for the garbage collector,
+	###   Reject any write attempts while using this database handle.
-	###   potentialy increasing the chance of unallocating storage
-	###   earlier.
 	###
-	### [:lifo_reclaim]
+	### [:writemap]
-	###   Recycle garbage collected items via LIFO, instead of FIFO.
+	###   Trade safety for speed for databases that fit within available
-	###   Depending on underlying hardware (disk write-back cache), this
+	###   memory. (See MDBX documentation for details.)
-	###   could increase write performance.
-	###
-	### [:no_metasync]
-	###   A system crash may sacrifice the last commit for a potentially
-	###   large write performance increase.  Database integrity is
-	###   maintained.
 	###
 	def self::open( *args, &block )
 		db = new( *args )

spec/mdbx/database_spec.rb

@@ -3,20 +3,18 @@
 
 require_relative '../lib/helper'
 
-
 RSpec.describe( MDBX::Database ) do
 
-	before( :all ) do
-		db = described_class.open( TEST_DATABASE.to_s )
-		db.clear
-		db.close
-	end
-
 	it "disallows direct calls to #new" do
 		expect{ described_class.new }.
 			to raise_exception( NoMethodError, /private/ )
 	end
 
+	it "raises an exception if passed unknown options" do
+		expect{ described_class.open( TEST_DATABASE.to_s, nope: true ) }.
+			to raise_exception( ArgumentError, /unknown option/i )
+	end
+
 	it "knows the env handle open/close state" do
 		db = described_class.open( TEST_DATABASE.to_s )
 		expect( db.closed? ).to be_falsey
@@ -52,12 +50,9 @@ RSpec.describe( MDBX::Database ) do
 
 		let!( :db ) { described_class.open( TEST_DATABASE.to_s ) }
 
-		before( :each ) do
-			db.clear
-		end
-
 		after( :each ) do
 			db.close
+			TEST_DATABASE.rmtree
 		end
 
 		it "can be reopened" do
@@ -86,12 +81,9 @@ RSpec.describe( MDBX::Database ) do
 
 		let!( :db ) { described_class.open( TEST_DATABASE.to_s ) }
 
-		before( :each ) do
-			db.clear
-		end
-
 		after( :each ) do
 			db.close
+			TEST_DATABASE.rmtree
 		end
 
 
@@ -181,16 +173,23 @@ RSpec.describe( MDBX::Database ) do
 	end
 
 
+	# context 'duplicate keys' do
+    #
+	# 	let( :db ) { described_class.open( TEST_DATABASE.to_s, max_collections: 5, duplicate_keys: true ) }
+    #
+	# 	after( :each ) do
+	# 		db.close
+	# 	end
+	# end
+
+
 	context 'collections' do
 
 		let!( :db ) { described_class.open( TEST_DATABASE.to_s, max_collections: 5 ) }
 
-		before( :each ) do
-			db.clear
-		end
-
 		after( :each ) do
 			db.close
+			TEST_DATABASE.rmtree
 		end
 
 		it "fail if the max_collections option is not specified when opening" do
@@ -324,6 +323,17 @@ RSpec.describe( MDBX::Database ) do
 			expect( db['doom'] ).to be_nil
 		end
 
+		it "retains other collections, only dropping what is specified" do
+			db.collection( 'boots' )
+			db.collection( 'pants' )
+			db.main
+			db.drop( 'boots' )
+
+			expect( db.collection ).to be_nil
+			expect( db['doom'] ).to be_nil
+			expect( db ).to have_key( :pants )
+		end
+
 		it "retains the collection environment when clearing data" do
 			db.collection( 'doom' )
 			db[ 'key' ] = 1
@@ -340,7 +350,7 @@ RSpec.describe( MDBX::Database ) do
 
 	context 'transactions' do
 
-		let!( :db ) { described_class.open( TEST_DATABASE.to_s, max_collections: 5 ) }
+		let( :db ) { described_class.open( TEST_DATABASE.to_s, max_collections: 5 ) }
 
 		before( :each ) do
 			db.clear
@@ -348,6 +358,7 @@ RSpec.describe( MDBX::Database ) do
 
 		after( :each ) do
 			db.close
+			TEST_DATABASE.rmtree
 		end
 
 		it "knows when a transaction is currently open" do
@@ -447,7 +458,7 @@ RSpec.describe( MDBX::Database ) do
 
 	context "iterators" do
 
-		let( :db ) {
+		let!( :db ) {
 			described_class.open( TEST_DATABASE.to_s, max_collections: 5 ).collection( 'iter' )
 		}
 
@@ -457,6 +468,7 @@ RSpec.describe( MDBX::Database ) do
 
 		after( :each ) do
 			db.close
+			TEST_DATABASE.rmtree
 		end
 
 		it "raises an exception if the caller didn't open a transaction first" do
@@ -496,12 +508,13 @@ RSpec.describe( MDBX::Database ) do
 
 	context "serialization" do
 
-		let( :db ) {
+		let!( :db ) {
 			described_class.open( TEST_DATABASE.to_s )
 		}
 
 		after( :each ) do
 			db.close
+			TEST_DATABASE.rmtree
 		end
 
 		it "uses Marshalling as default" do