author | Mahlon E. Smith <mahlon@laika.com> |
Thu, 05 Mar 2020 12:01:31 -0800 | |
changeset 21 | 1b9ceef22793 |
parent 20 | e2e96d97b77c |
permissions | -rw-r--r-- |
0 | 1 |
# -*- ruby -*- |
20 | 2 |
# frozen_string_literal: true |
0 | 3 |
|
4 |
require 'loggability' |
|
5 |
require 'configurability' |
|
6 |
require 'sequel' |
|
7 |
require 'strelka' |
|
8 |
require 'strelka/mixins' |
|
9 |
||
10 |
require 'thingfish' |
|
11 |
require 'thingfish/mixins' |
|
12 |
require 'thingfish/metastore' |
|
13 |
||
14 |
# Toplevel namespace |
|
15 |
class Thingfish::Metastore::PgGraph < Thingfish::Metastore |
|
16 |
extend Loggability, |
|
17 |
Configurability, |
|
18 |
Strelka::MethodUtilities |
|
19 |
include Thingfish::Normalization |
|
20 |
||
21 |
||
22 |
# Load Sequel extensions/plugins |
|
23 |
Sequel.extension :migration |
|
24 |
||
25 |
||
26 |
# Package version |
|
20 | 27 |
VERSION = '0.5.0' |
0 | 28 |
|
29 |
# Version control revision |
|
15 | 30 |
REVISION = %q$Revision: 686fbfe638bd $ |
0 | 31 |
|
32 |
# The data directory that contains migration files. |
|
33 |
# |
|
34 |
DATADIR = if ENV['THINGFISH_METASTORE_PGGRAPH_DATADIR'] |
|
16 | 35 |
Pathname( ENV['THINGFISH_METASTORE_PGGRAPH_DATADIR'] ) |
15 | 36 |
elsif Gem.loaded_specs[ 'thingfish-metastore-pggraph' ] && File.exist?( Gem.loaded_specs['thingfish-metastore-pggraph'].datadir ) |
37 |
Pathname( Gem.loaded_specs['thingfish-metastore-pggraph'].datadir ) |
|
38 |
else |
|
39 |
Pathname( __FILE__ ).dirname.parent.parent.parent + 'data/thingfish-metastore-pggraph' |
|
40 |
end |
|
0 | 41 |
|
42 |
log_as :thingfish_metastore_pggraph |
|
43 |
||
44 |
# Configurability API -- load the `pg_metastore` |
|
11
b1f97f6063a3
Update for Configurability 3.x.
Mahlon E. Smith <mahlon@martini.nu>
parents:
8
diff
changeset
|
45 |
configurability 'thingfish.pggraph_metastore' do |
b1f97f6063a3
Update for Configurability 3.x.
Mahlon E. Smith <mahlon@martini.nu>
parents:
8
diff
changeset
|
46 |
|
b1f97f6063a3
Update for Configurability 3.x.
Mahlon E. Smith <mahlon@martini.nu>
parents:
8
diff
changeset
|
47 |
# The Sequel::Database that's used to access the metastore tables |
b1f97f6063a3
Update for Configurability 3.x.
Mahlon E. Smith <mahlon@martini.nu>
parents:
8
diff
changeset
|
48 |
setting :uri, default: 'postgres:/thingfish' |
0 | 49 |
|
11
b1f97f6063a3
Update for Configurability 3.x.
Mahlon E. Smith <mahlon@martini.nu>
parents:
8
diff
changeset
|
50 |
# The number of seconds to consider a "slow" query |
b1f97f6063a3
Update for Configurability 3.x.
Mahlon E. Smith <mahlon@martini.nu>
parents:
8
diff
changeset
|
51 |
setting :slow_query_seconds, default: 0.01 |
b1f97f6063a3
Update for Configurability 3.x.
Mahlon E. Smith <mahlon@martini.nu>
parents:
8
diff
changeset
|
52 |
end |
0 | 53 |
|
54 |
# The Sequel::Database that's used to access the metastore tables |
|
55 |
singleton_attr_accessor :db |
|
56 |
||
57 |
||
58 |
### Set up the metastore database and migrate to the latest version. |
|
59 |
def self::setup_database |
|
60 |
Sequel.extension :pg_json_ops |
|
13
686fbfe638bd
Remove deprecated calls for the Sequel ORM, add compatibility for Sequel 5.
Mahlon E. Smith <mahlon@laika.com>
parents:
11
diff
changeset
|
61 |
Sequel.split_symbols = true |
686fbfe638bd
Remove deprecated calls for the Sequel ORM, add compatibility for Sequel 5.
Mahlon E. Smith <mahlon@laika.com>
parents:
11
diff
changeset
|
62 |
Sequel::Model.require_valid_table = false |
0 | 63 |
|
64 |
self.db = Sequel.connect( self.uri ) |
|
65 |
self.db.logger = Loggability[ Thingfish::Metastore::PgGraph ] |
|
66 |
self.db.extension :pg_streaming |
|
20 | 67 |
self.db.extension :pg_inet |
0 | 68 |
self.db.stream_all_queries = true |
69 |
self.db.sql_log_level = :debug |
|
70 |
self.db.extension( :pg_json ) |
|
71 |
self.db.log_warn_duration = self.slow_query_seconds |
|
72 |
||
73 |
# Ensure the database is current. |
|
74 |
# |
|
75 |
unless Sequel::Migrator.is_current?( self.db, self.migrations_dir.to_s ) |
|
76 |
self.log.info "Installing database schema..." |
|
77 |
Sequel::Migrator.apply( self.db, self.migrations_dir.to_s ) |
|
78 |
end |
|
79 |
end |
|
80 |
||
81 |
||
82 |
### Tear down the configured metastore database. |
|
83 |
def self::teardown_database |
|
84 |
self.log.info "Tearing down database schema..." |
|
85 |
Sequel::Migrator.apply( self.db, self.migrations_dir.to_s, 0 ) |
|
86 |
end |
|
87 |
||
88 |
||
89 |
### Return the current database migrations directory as a Pathname |
|
90 |
def self::migrations_dir |
|
91 |
return DATADIR + 'migrations' |
|
92 |
end |
|
93 |
||
94 |
||
95 |
### Configurability API -- set up the metastore with the `pg_metastore` section of |
|
96 |
### the config file. |
|
97 |
def self::configure( config=nil ) |
|
11
b1f97f6063a3
Update for Configurability 3.x.
Mahlon E. Smith <mahlon@martini.nu>
parents:
8
diff
changeset
|
98 |
super |
b1f97f6063a3
Update for Configurability 3.x.
Mahlon E. Smith <mahlon@martini.nu>
parents:
8
diff
changeset
|
99 |
self.setup_database if self.uri |
0 | 100 |
end |
101 |
||
102 |
||
103 |
### Set up the metastore. |
|
104 |
def initialize( * ) # :notnew: |
|
105 |
require 'thingfish/metastore/pggraph/node' |
|
106 |
require 'thingfish/metastore/pggraph/edge' |
|
13
686fbfe638bd
Remove deprecated calls for the Sequel ORM, add compatibility for Sequel 5.
Mahlon E. Smith <mahlon@laika.com>
parents:
11
diff
changeset
|
107 |
|
686fbfe638bd
Remove deprecated calls for the Sequel ORM, add compatibility for Sequel 5.
Mahlon E. Smith <mahlon@laika.com>
parents:
11
diff
changeset
|
108 |
Thingfish::Metastore::PgGraph::Node.dataset = self.class.db[ :nodes ] |
686fbfe638bd
Remove deprecated calls for the Sequel ORM, add compatibility for Sequel 5.
Mahlon E. Smith <mahlon@laika.com>
parents:
11
diff
changeset
|
109 |
Thingfish::Metastore::PgGraph::Edge.dataset = self.class.db[ :edges ] |
0 | 110 |
@model = Thingfish::Metastore::PgGraph::Node |
111 |
end |
|
112 |
||
113 |
||
114 |
###### |
|
115 |
public |
|
116 |
###### |
|
117 |
||
118 |
## |
|
119 |
# The Sequel model representing the metadata rows. |
|
120 |
attr_reader :model |
|
121 |
||
122 |
||
123 |
# |
|
124 |
# :section: Thingfish::Metastore API |
|
125 |
# |
|
126 |
||
127 |
### Return an Array of all stored oids. |
|
128 |
def oids |
|
129 |
return self.each_oid.to_a |
|
130 |
end |
|
131 |
||
132 |
||
133 |
### Iterate over each of the store's oids, yielding to the block if one is given |
|
134 |
### or returning an Enumerator if one is not. |
|
135 |
def each_oid( &block ) |
|
136 |
return self.model.select_map( :id ).each( &block ) |
|
137 |
end |
|
138 |
||
139 |
||
140 |
### Save the +metadata+ Hash for the specified +oid+. |
|
141 |
def save( oid, metadata ) |
|
142 |
md = self.model.from_hash( metadata ) |
|
143 |
md.id = oid |
|
144 |
md.save |
|
145 |
end |
|
146 |
||
147 |
||
148 |
### Fetch the data corresponding to the given +oid+ as a Hash-ish object. |
|
149 |
def fetch( oid, *keys ) |
|
150 |
metadata = self.model[ oid ] or return nil |
|
151 |
||
152 |
if keys.empty? |
|
153 |
return metadata.to_hash |
|
154 |
else |
|
155 |
keys = normalize_keys( keys ) |
|
156 |
values = metadata.to_hash.values_at( *keys ) |
|
157 |
return Hash[ [keys, values].transpose ] |
|
158 |
end |
|
159 |
end |
|
160 |
||
161 |
||
162 |
### Fetch the value of the metadata associated with the given +key+ for the |
|
163 |
### specified +oid+. |
|
164 |
def fetch_value( oid, key ) |
|
165 |
metadata = self.model[ oid ] or return nil |
|
3
5b4ee98d698c
Pull recent Metastore::PG fixes into Metastore::PgGraph.
Mahlon E. Smith <mahlon@martini.nu>
parents:
1
diff
changeset
|
166 |
key = key.to_sym |
5b4ee98d698c
Pull recent Metastore::PG fixes into Metastore::PgGraph.
Mahlon E. Smith <mahlon@martini.nu>
parents:
1
diff
changeset
|
167 |
return metadata[ key ] || metadata.user_metadata[ key ] |
0 | 168 |
end |
169 |
||
170 |
||
171 |
### Fetch UUIDs related to the given +oid+. |
|
172 |
def fetch_related_oids( oid ) |
|
173 |
oid = normalize_oid( oid ) |
|
174 |
oid = self.model[ oid ] |
|
175 |
return [] unless oid |
|
176 |
return oid.related_nodes.map( &:id_c ) |
|
177 |
end |
|
178 |
||
179 |
||
180 |
### Search the metastore for UUIDs which match the specified +criteria+ and |
|
181 |
### return them as an iterator. |
|
182 |
def search( options={} ) |
|
183 |
ds = self.model.naked.select( :id ) |
|
184 |
self.log.debug "Starting search with %p" % [ ds ] |
|
185 |
||
186 |
ds = self.omit_related_resources( ds, options ) |
|
187 |
ds = self.apply_search_criteria( ds, options ) |
|
188 |
ds = self.apply_search_order( ds, options ) |
|
189 |
ds = self.apply_search_direction( ds, options ) |
|
190 |
ds = self.apply_search_limit( ds, options ) |
|
191 |
||
3
5b4ee98d698c
Pull recent Metastore::PG fixes into Metastore::PgGraph.
Mahlon E. Smith <mahlon@martini.nu>
parents:
1
diff
changeset
|
192 |
self.log.debug "Dataset for search is: %s" % [ ds.sql ] |
5b4ee98d698c
Pull recent Metastore::PG fixes into Metastore::PgGraph.
Mahlon E. Smith <mahlon@martini.nu>
parents:
1
diff
changeset
|
193 |
|
0 | 194 |
return ds.map {|row| row[:id] } |
195 |
end |
|
196 |
||
197 |
||
198 |
### Update the metadata for the given +oid+ with the specified +values+ hash. |
|
199 |
def merge( oid, values ) |
|
200 |
values = normalize_keys( values ) |
|
201 |
||
202 |
md = self.model[ oid ] or return nil |
|
203 |
md.merge!( values ) |
|
204 |
md.save |
|
205 |
end |
|
206 |
||
207 |
||
208 |
### Remove all metadata associated with +oid+ from the Metastore. |
|
209 |
def remove( oid, *keys ) |
|
210 |
self.model[ id: oid ].destroy |
|
211 |
end |
|
212 |
||
213 |
||
214 |
### Remove all metadata associated with +oid+ except for the specified +keys+. |
|
215 |
def remove_except( oid, *keys ) |
|
216 |
keys = normalize_keys( keys ) |
|
217 |
||
218 |
md = self.model[ oid ] or return nil |
|
219 |
md.user_metadata.keep_if {|key,_| keys.include?(key) } |
|
220 |
md.save |
|
221 |
end |
|
222 |
||
223 |
||
224 |
### Returns +true+ if the metastore has metadata associated with the specified +oid+. |
|
225 |
def include?( oid ) |
|
226 |
return self.model.count( id: oid ).nonzero? |
|
227 |
end |
|
228 |
||
229 |
||
230 |
### Returns the number of objects the store contains. |
|
231 |
def size |
|
232 |
return self.model.count |
|
233 |
end |
|
234 |
||
235 |
||
236 |
######### |
|
237 |
protected |
|
238 |
######### |
|
239 |
||
240 |
### Omit related resources from the search dataset +ds+ unless the given |
|
241 |
### +options+ specify otherwise. |
|
242 |
def omit_related_resources( ds, options ) |
|
243 |
unless options[:include_related] |
|
244 |
self.log.debug " omitting entries for related resources" |
|
245 |
ds = ds.unrelated |
|
246 |
end |
|
247 |
return ds |
|
248 |
end |
|
249 |
||
250 |
||
251 |
### Apply the search :criteria from the specified +options+ to the collection |
|
252 |
### in +ds+ and return the modified dataset. |
|
253 |
def apply_search_criteria( ds, options ) |
|
254 |
if (( criteria = options[:criteria] )) |
|
255 |
criteria.each do |field, value| |
|
256 |
self.log.debug " applying criteria: %p => %p" % [ field.to_s, value ] |
|
257 |
ds = ds.where_metadata( field => value ) |
|
258 |
end |
|
259 |
end |
|
260 |
||
261 |
return ds |
|
262 |
end |
|
263 |
||
264 |
||
265 |
### Apply the search :order from the specified +options+ to the collection in |
|
266 |
### +ds+ and return the modified dataset. |
|
267 |
def apply_search_order( ds, options ) |
|
268 |
if options[:order] |
|
269 |
columns = Array( options[:order] ) |
|
3
5b4ee98d698c
Pull recent Metastore::PG fixes into Metastore::PgGraph.
Mahlon E. Smith <mahlon@martini.nu>
parents:
1
diff
changeset
|
270 |
self.log.debug " ordering results by columns: %p" % [ columns ] |
5b4ee98d698c
Pull recent Metastore::PG fixes into Metastore::PgGraph.
Mahlon E. Smith <mahlon@martini.nu>
parents:
1
diff
changeset
|
271 |
ds = ds.order_metadata( columns ) |
0 | 272 |
end |
273 |
||
274 |
return ds |
|
275 |
end |
|
276 |
||
277 |
||
278 |
### Apply the search :direction from the specified +options+ to the collection |
|
279 |
### in +ds+ and return the modified dataset. |
|
280 |
def apply_search_direction( ds, options ) |
|
281 |
ds = ds.reverse if options[:direction] && options[:direction] == 'desc' |
|
282 |
return ds |
|
283 |
end |
|
284 |
||
285 |
||
286 |
### Apply the search :limit from the specified +options+ to the collection in |
|
287 |
### +ds+ and return the modified dataset. |
|
288 |
def apply_search_limit( ds, options ) |
|
289 |
if (( limit = options[:limit] )) |
|
290 |
self.log.debug " limiting to %s results" % [ limit ] |
|
291 |
offset = options[:offset] || 0 |
|
292 |
ds = ds.limit( limit, offset ) |
|
293 |
end |
|
294 |
||
295 |
return ds |
|
296 |
end |
|
297 |
||
18
e18bc5021028
Support forking handlers using this metastore.
Mahlon E. Smith <mahlon@martini.nu>
parents:
16
diff
changeset
|
298 |
|
e18bc5021028
Support forking handlers using this metastore.
Mahlon E. Smith <mahlon@martini.nu>
parents:
16
diff
changeset
|
299 |
# Register a hook to make the database connection fork-safe. |
e18bc5021028
Support forking handlers using this metastore.
Mahlon E. Smith <mahlon@martini.nu>
parents:
16
diff
changeset
|
300 |
Strelka.before_fork do |
e18bc5021028
Support forking handlers using this metastore.
Mahlon E. Smith <mahlon@martini.nu>
parents:
16
diff
changeset
|
301 |
self.db&.disconnect |
e18bc5021028
Support forking handlers using this metastore.
Mahlon E. Smith <mahlon@martini.nu>
parents:
16
diff
changeset
|
302 |
end |
e18bc5021028
Support forking handlers using this metastore.
Mahlon E. Smith <mahlon@martini.nu>
parents:
16
diff
changeset
|
303 |
|
0 | 304 |
end # class Thingfish::Metastore::PgGraph |
305 |