/__w/shurbej/shurbej/_build/test/cover/aggregate/shurbej_http_items.html

1 -module(shurbej_http_items).
2 -include_lib("shurbej_store/include/shurbej_records.hrl").
3
4 -export([init/2, generate_key/0, validate_each/2, envelope_for_write/3]).
5
6 -define(MAX_WRITE_ITEMS, 50).
7
8 init(Req0, State) ->
9 130 case shurbej_http_common:authorize(Req0) of
10 {ok, LibRef, _} ->
11 127 Method = cowboy_req:method(Req0),
12 127 Perm = perm_for_method(Method),
13 127 case shurbej_http_common:check_lib_perm(Perm, LibRef) of
14 {error, forbidden} ->
15 4 Req = shurbej_http_common:error_response(403, <<"Access denied">>, Req0),
16 4 {ok, Req, State};
17 ok ->
18 123 handle(Method, Req0, State)
19 end;
20 {error, bad_request, Req} ->
21 
:-(
 Req2 = shurbej_http_common:error_response(400, <<"Invalid user ID">>, Req),
22 
:-(
 {ok, Req2, State};
23 {error, Reason, _} ->
24 3 Req = shurbej_http_common:auth_error_response(Reason, Req0),
25 3 {ok, Req, State}
26 end.
27
28 53 perm_for_method(<<"GET">>) -> read;
29 
:-(
 perm_for_method(<<"HEAD">>) -> read;
30 74 perm_for_method(_) -> write.
31
32 %% GET single item
33 handle(<<"GET">>, Req0, #{scope := single} = State) ->
34 4 LibRef = shurbej_http_common:lib_ref(Req0),
35 4 ItemKey = cowboy_req:binding(item_key, Req0),
36 4 case shurbej_db:get_item(LibRef, ItemKey) of
37 {ok, Item} ->
38 3 {ok, LibVersion} = shurbej_version:get(LibRef),
39 3 ChildrenCounts = cached_children_counts(LibRef, LibVersion),
40 3 Body = shurbej_http_common:envelope_item(LibRef, Item, ChildrenCounts),
41 3 Req = shurbej_http_common:json_response(200, Body, Item#shurbej_item.version, Req0),
42 3 {ok, Req, State};
43 undefined ->
44 1 Req = shurbej_http_common:error_response(404, <<"Item not found">>, Req0),
45 1 {ok, Req, State}
46 end;
47
48 %% GET list — dispatch by scope
49 handle(<<"GET">>, Req0, #{scope := Scope} = State) ->
50 47 LibRef = shurbej_http_common:lib_ref(Req0),
51 47 Since = shurbej_http_common:get_since(Req0),
52 47 {ok, LibVersion} = shurbej_version:get(LibRef),
53 47 case shurbej_http_common:check_304(Req0, LibVersion) of
54 {304, Req} ->
55 1 {ok, Req, State};
56 continue ->
57 46 Items = fetch_items(Scope, LibRef, Since, Req0),
58 46 Items2 = apply_filters(Items, Req0),
59 46 Format = shurbej_http_common:get_format(Req0),
60 46 respond_list(Format, Items2, LibRef, LibVersion, Req0, State)
61 end;
62
63 %% POST — create/update items (with write token idempotency)
64 handle(<<"POST">>, Req0, State) ->
65 62 WriteToken = shurbej_http_common:get_write_token(Req0),
66 62 case shurbej_write_token:check(WriteToken) of
67 {duplicate, {Result, Version}} ->
68 1 Req = shurbej_http_common:json_response(200, Result, Version, Req0),
69 1 {ok, Req, State};
70 in_progress ->
71 %% Concurrent request with same token — proceed without token caching
72 
:-(
 do_post(Req0, State, undefined);
73 new ->
74 61 do_post(Req0, State, WriteToken)
75 end;
76
77 %% PUT/PATCH single item — update a specific item
78 handle(Method, Req0, #{scope := single} = State) when Method =:= <<"PUT">>; Method =:= <<"PATCH">> ->
79 3 LibRef = shurbej_http_common:lib_ref(Req0),
80 3 ItemKey = cowboy_req:binding(item_key, Req0),
81 3 ExpectedVersion = shurbej_http_common:get_if_unmodified(Req0),
82 3 case shurbej_http_common:read_json_body(Req0) of
83 {error, Reason, Req1} ->
84 
:-(
 Req = shurbej_http_common:error_response(400, body_error(Reason), Req1),
85 
:-(
 {ok, Req, State};
86 {ok, Incoming, Req1} ->
87 3 case shurbej_db:get_item(LibRef, ItemKey) of
88 undefined ->
89 1 Req = shurbej_http_common:error_response(404, <<"Item not found">>, Req1),
90 1 {ok, Req, State};
91 {ok, #shurbej_item{data = Existing}} ->
92 2 Merged = case Method of
93 1 <<"PATCH">> -> maps:merge(Existing, Incoming);
94 1 <<"PUT">> -> Incoming
95 end,
96 2 Item = Merged#{<<"key">> => ItemKey},
97 2 case shurbej_validate:item(Item) of
98 {error, Reason2} ->
99 
:-(
 Req = shurbej_http_common:error_response(400, Reason2, Req1),
100 
:-(
 {ok, Req, State};
101 ok ->
102 2 case shurbej_version:write(LibRef, ExpectedVersion, fun(NewVersion) ->
103 2 write_item(LibRef, Item, NewVersion)
104 end) of
105 {ok, NewVersion} ->
106 2 Envelope = envelope_for_write(LibRef, Item, NewVersion),
107 2 Req = shurbej_http_common:json_response(200, Envelope, NewVersion, Req1),
108 2 {ok, Req, State};
109 {error, precondition, CurrentVersion} ->
110 
:-(
 Req = shurbej_http_common:json_response(412,
111 #{<<"message">> => <<"Library has been modified since specified version">>},
112 CurrentVersion, Req1),
113 
:-(
 {ok, Req, State}
114 end
115 end
116 end
117 end;
118
119 %% DELETE single item
120 handle(<<"DELETE">>, Req0, #{scope := single} = State) ->
121 1 LibRef = shurbej_http_common:lib_ref(Req0),
122 1 ItemKey = cowboy_req:binding(item_key, Req0),
123 1 ExpectedVersion = shurbej_http_common:get_if_unmodified(Req0),
124 1 case shurbej_version:write(LibRef, ExpectedVersion, fun(NewVersion) ->
125 1 cascade_delete(LibRef, ItemKey, NewVersion),
126 1 ok
127 end) of
128 {ok, NewVersion} ->
129 1 Req = cowboy_req:reply(204, #{
130 <<"last-modified-version">> => integer_to_binary(NewVersion)
131 }, Req0),
132 1 {ok, Req, State};
133 {error, precondition, CurrentVersion} ->
134 
:-(
 Req = shurbej_http_common:json_response(412,
135 #{<<"message">> => <<"Library has been modified since specified version">>},
136 CurrentVersion, Req0),
137 
:-(
 {ok, Req, State}
138 end;
139
140 %% DELETE multiple — cascade to tags, fulltext, file metadata, children
141 handle(<<"DELETE">>, Req0, State) ->
142 6 LibRef = shurbej_http_common:lib_ref(Req0),
143 6 ExpectedVersion = shurbej_http_common:get_if_unmodified(Req0),
144 6 #{itemKey := KeysParam} = cowboy_req:match_qs([{itemKey, [], <<>>}], Req0),
145 6 Keys = [K || K <- binary:split(KeysParam, <<",">>, [global]), K =/= <<>>],
146 6 case Keys of
147 [] ->
148 
:-(
 Req = shurbej_http_common:error_response(400, <<"No item keys specified">>, Req0),
149 
:-(
 {ok, Req, State};
150 _ ->
151 6 case shurbej_version:write(LibRef, ExpectedVersion, fun(NewVersion) ->
152 5 lists:foreach(fun(K) ->
153 5 cascade_delete(LibRef, K, NewVersion)
154 end, Keys),
155 5 ok
156 end) of
157 {ok, NewVersion} ->
158 5 Req = cowboy_req:reply(204, #{
159 <<"last-modified-version">> => integer_to_binary(NewVersion)
160 }, Req0),
161 5 {ok, Req, State};
162 {error, precondition, CurrentVersion} ->
163 1 Req = shurbej_http_common:json_response(412,
164 #{<<"message">> => <<"Library has been modified since specified version">>},
165 CurrentVersion, Req0),
166 1 {ok, Req, State}
167 end
168 end;
169
170 handle(_, Req0, State) ->
171 
:-(
 Req = shurbej_http_common:error_response(405, <<"Method not allowed">>, Req0),
172 
:-(
 {ok, Req, State}.
173
174 %% Internal — respond to list GET by format
175
176 respond_list(<<"versions">>, Items, _LibRef, LibVersion, Req0, State) ->
177 7 VersionMap = maps:from_list(
178 84 [{K, V} || #shurbej_item{id = {_, _, K}, version = V} <- Items]),
179 7 Req = shurbej_http_common:json_response(200, VersionMap, LibVersion, Req0),
180 7 {ok, Req, State};
181 respond_list(<<"keys">>, Items, _LibRef, LibVersion, Req0, State) ->
182 1 Keys = [K || #shurbej_item{id = {_, _, K}} <- Items],
183 1 Req = shurbej_http_common:json_response(200, Keys, LibVersion, Req0),
184 1 {ok, Req, State};
185 respond_list(Format, _Items, _LibRef, _LibVersion, Req0, State)
186 when Format =:= <<"atom">>; Format =:= <<"bib">>;
187 Format =:= <<"ris">>; Format =:= <<"mods">> ->
188 1 Req = shurbej_http_common:error_response(400,
189 <<"Export format '", Format/binary, "' is not supported by this server">>, Req0),
190 1 {ok, Req, State};
191 respond_list(_, Items, LibRef, LibVersion, Req0, State) ->
192 37 ChildrenCounts = cached_children_counts(LibRef, LibVersion),
193 37 Sorted = shurbej_http_common:sort_records(Items,
194 shurbej_http_common:get_sort(Req0),
195 shurbej_http_common:get_direction(Req0)),
196 37 Req = shurbej_http_common:list_response(Req0, Sorted, LibVersion,
197 443 fun(I) -> shurbej_http_common:envelope_item(LibRef, I, ChildrenCounts) end),
198 37 {ok, Req, State}.
199
200 %% Internal — fetch items by scope
201
202 fetch_items(all, LibRef, Since, Req) ->
203 43 Base = shurbej_db:list_items(LibRef, Since),
204 43 maybe_include_trashed(Base, LibRef, Since, Req);
205 fetch_items(top, LibRef, Since, Req) ->
206 1 Base = shurbej_db:list_items_top(LibRef, Since),
207 1 maybe_include_trashed(Base, LibRef, Since, Req);
208 fetch_items(trash, LibRef, Since, _Req) ->
209 1 shurbej_db:list_items_trash(LibRef, Since);
210 fetch_items(children, LibRef, Since, Req) ->
211 1 ParentKey = cowboy_req:binding(item_key, Req),
212 1 shurbej_db:list_items_children(LibRef, ParentKey, Since);
213 fetch_items(collection, LibRef, Since, Req) ->
214 
:-(
 CollKey = cowboy_req:binding(coll_key, Req),
215 
:-(
 shurbej_db:list_items_in_collection(LibRef, CollKey, Since);
216 fetch_items(collection_top, LibRef, Since, Req) ->
217 
:-(
 CollKey = cowboy_req:binding(coll_key, Req),
218 
:-(
 [I || #shurbej_item{data = D} = I
219 
:-(
 <- shurbej_db:list_items_in_collection(LibRef, CollKey, Since),
220 
:-(
 maps:get(<<"parentItem">>, D, false) =:= false].
221
222 maybe_include_trashed(Items, LibRef, Since, Req) ->
223 44 case shurbej_http_common:get_include_trashed(Req) of
224 1 true -> Items ++ shurbej_db:list_items_trash(LibRef, Since);
225 43 false -> Items
226 end.
227
228 apply_filters(Items, Req) ->
229 46 Items2 = filter_by_item_keys(Items, shurbej_http_common:get_item_keys(Req)),
230 46 Items3 = shurbej_http_common:filter_by_tag(Items2, shurbej_http_common:get_tag_filter(Req)),
231 46 Items4 = shurbej_http_common:filter_by_item_type(Items3, shurbej_http_common:get_item_type_filter(Req)),
232 46 shurbej_http_common:filter_by_query(Items4, shurbej_http_common:get_query(Req),
233 shurbej_http_common:get_qmode(Req)).
234
235 46 filter_by_item_keys(Items, all) -> Items;
236 filter_by_item_keys(Items, Keys) ->
237 
:-(
 KeySet = sets:from_list(Keys),
238 
:-(
 [I || #shurbej_item{id = {_, _, K}} = I <- Items, sets:is_element(K, KeySet)].
239
240 do_post(Req0, State, WriteToken) ->
241 61 LibRef = shurbej_http_common:lib_ref(Req0),
242 61 ExpectedVersion = shurbej_http_common:get_if_unmodified(Req0),
243 61 case shurbej_http_common:read_json_body(Req0) of
244 {error, Reason, Req1} ->
245 1 Req = shurbej_http_common:error_response(400, body_error(Reason), Req1),
246 1 {ok, Req, State};
247 {ok, Items, Req1} when is_list(Items), length(Items) =< ?MAX_WRITE_ITEMS ->
248 60 KeyedItems = [ensure_key(I) || I <- Items],
249 60 {Valid, Failed} = validate_each(KeyedItems, fun shurbej_validate:item/1),
250 60 case Valid of
251 [] when map_size(Failed) > 0 ->
252 6 Result = #{<<"successful">> => #{}, <<"unchanged">> => #{}, <<"failed">> => Failed},
253 6 Req = shurbej_http_common:json_response(400, Result, Req1),
254 6 {ok, Req, State};
255 _ ->
256 54 case shurbej_version:write(LibRef, ExpectedVersion, fun(NewVersion) ->
257 53 lists:foreach(fun({_Idx, Item}) ->
258 66 write_item(LibRef, Item, NewVersion)
259 end, Valid),
260 53 ok
261 end) of
262 {ok, NewVersion} ->
263 53 Successful = maps:from_list(
264 66 [{integer_to_binary(Idx), envelope_for_write(LibRef, Item, NewVersion)}
265 53 || {Idx, Item} <- Valid]),
266 53 Result = #{
267 <<"successful">> => Successful,
268 <<"unchanged">> => #{},
269 <<"failed">> => Failed
270 },
271 53 shurbej_write_token:store(WriteToken, {Result, NewVersion}),
272 53 Req = shurbej_http_common:json_response(200, Result, NewVersion, Req1),
273 53 {ok, Req, State};
274 {error, precondition, CurrentVersion} ->
275 1 Req = shurbej_http_common:json_response(412,
276 #{<<"message">> => <<"Library has been modified since specified version">>},
277 CurrentVersion, Req1),
278 1 {ok, Req, State}
279 end
280 end;
281 {ok, Items, Req1} when is_list(Items) ->
282 
:-(
 Req = shurbej_http_common:error_response(413,
283 <<"Too many items (max ", (integer_to_binary(?MAX_WRITE_ITEMS))/binary, ")">>, Req1),
284 
:-(
 {ok, Req, State};
285 {ok, _, Req1} ->
286 
:-(
 Req = shurbej_http_common:error_response(400, <<"Body must be a JSON array">>, Req1),
287 
:-(
 {ok, Req, State}
288 end.
289
290 %% Cascade delete — wrapped in Mnesia transaction for atomicity
291 cascade_delete(LibRef, ItemKey, NewVersion) ->
292 6 shurbej_db:mark_item_deleted(LibRef, ItemKey, NewVersion),
293 6 shurbej_db:record_deletion(LibRef, <<"item">>, ItemKey, NewVersion),
294 6 shurbej_db:delete_item_tags(LibRef, ItemKey),
295 6 shurbej_db:delete_item_collections(LibRef, ItemKey),
296 6 shurbej_db:delete_fulltext(LibRef, ItemKey),
297 6 shurbej_db:delete_file_meta(LibRef, ItemKey),
298 %% list_items_children now uses the parent_key secondary index — O(k) not O(n).
299 6 Children = shurbej_db:list_items_children(LibRef, ItemKey, 0),
300 6 lists:foreach(fun(#shurbej_item{id = {_, _, ChildKey}}) ->
301 
:-(
 cascade_delete(LibRef, ChildKey, NewVersion)
302 end, Children).
303
304 ensure_key(Item) when is_map(Item) ->
305 74 case maps:get(<<"key">>, Item, undefined) of
306 72 undefined -> Item#{<<"key">> => generate_key()};
307 2 _ -> Item
308 end;
309 
:-(
 ensure_key(_) -> #{<<"key">> => generate_key()}.
310
311 write_item({LT, LI} = LibRef, Item, NewVersion) ->
312 68 Key = maps:get(<<"key">>, Item),
313 68 FullData = Item#{<<"version">> => NewVersion},
314 68 ParentKey = case maps:get(<<"parentItem">>, Item, false) of
315 3 P when is_binary(P) -> P;
316 65 _ -> undefined
317 end,
318 68 shurbej_db:write_item(#shurbej_item{
319 id = {LT, LI, Key},
320 version = NewVersion,
321 data = FullData,
322 deleted = false,
323 parent_key = ParentKey
324 }),
325 68 Collections = maps:get(<<"collections">>, Item, []),
326 68 shurbej_db:set_item_collections(LibRef, Key, Collections),
327 68 Tags = maps:get(<<"tags">>, Item, []),
328 68 TagPairs = [{maps:get(<<"tag">>, T, <<>>), maps:get(<<"type">>, T, 0)} || T <- Tags],
329 68 shurbej_db:set_item_tags(LibRef, Key, TagPairs).
330
331 validate_each(Items, ValidateFn) ->
332 74 {Valid, Failed} = lists:foldl(fun({Idx, Item}, {V, F}) ->
333 90 case ValidateFn(Item) of
334 81 ok -> {[{Idx, Item} | V], F};
335 {error, Reason} ->
336 9 F2 = F#{integer_to_binary(Idx) => #{
337 <<"key">> => maps:get(<<"key">>, Item, <<>>),
338 <<"code">> => 400,
339 <<"message">> => Reason
340 }},
341 9 {V, F2}
342 end
343 end, {[], #{}}, lists:enumerate(0, Items)),
344 74 {lists:reverse(Valid), Failed}.
345
346 envelope_for_write(LibRef, Item, NewVersion) ->
347 80 Key = maps:get(<<"key">>, Item),
348 80 Base = shurbej_http_common:base_url(),
349 80 Prefix = shurbej_http_common:lib_path_prefix(LibRef),
350 80 FullData = Item#{<<"key">> => Key, <<"version">> => NewVersion},
351 80 #{
352 <<"key">> => Key,
353 <<"version">> => NewVersion,
354 <<"library">> => shurbej_http_common:library_obj(LibRef),
355 <<"links">> => #{
356 <<"self">> => #{
357 <<"href">> => <<Base/binary, Prefix/binary, "/items/", Key/binary>>,
358 <<"type">> => <<"application/json">>
359 }
360 },
361 <<"meta">> => #{<<"numChildren">> => 0},
362 <<"data">> => FullData
363 }.
364
365 %% Generate item key using CSPRNG — not rand:uniform
366 generate_key() ->
367 88 Chars = <<"23456789ABCDEFGHIJKLMNPQRSTUVWXYZ">>,
368 88 Len = byte_size(Chars),
369 88 Bytes = crypto:strong_rand_bytes(8),
370 88 list_to_binary([binary:at(Chars, B rem Len) || <<B>> <= Bytes]).
371
372 1 body_error(invalid_json) -> <<"Invalid JSON">>;
373 
:-(
 body_error(body_too_large) -> <<"Request body too large">>.
374
375 %% One cached {LibVersion, Counts} row per library in a public ETS table
376 %% owned by shurbej_rate_limit. A version mismatch recomputes and overwrites.
377 %% persistent_term would be wrong here — each write bumps LibVersion and
378 %% `persistent_term:put` triggers a global scan across all terms.
379 cached_children_counts(LibRef, LibVersion) ->
380 40 case ets:lookup(shurbej_children_cache, LibRef) of
381 15 [{_, LibVersion, Counts}] -> Counts;
382 _ ->
383 25 Counts = shurbej_db:count_item_children(LibRef),
384 25 ets:insert(shurbej_children_cache, {LibRef, LibVersion, Counts}),
385 25 Counts
386 end.
Line Hits Source