feat(operations): remove attributes over the operations API (full_record, __unset__)

dataLayer/harperBridge/ResourceBridge.ts

@@ -8,6 +8,7 @@ import {
 	VALUE_SEARCH_COMPARATORS,
 	VALUE_SEARCH_COMPARATORS_REVERSE_LOOKUP,
 	READ_AUDIT_LOG_SEARCH_TYPES_ENUM,
+	UNSET_ATTRIBUTES as OPERATIONS_UNSET_KEY,
 } from '../../utility/hdbTerms.ts';
 import * as signalling from '../../utility/signalling.ts';
 import { SchemaEventMsg } from '../../server/threads/itc.js';
@@ -203,6 +204,23 @@ export class ResourceBridge extends BridgeMethods {
 		const { attributes } = insertUpdateValidate(upsertObj);
 
 		let new_attributes;
+		// `full_record: true` makes this a full replace instead of a merge: an attribute absent from
+		// the submitted record is REMOVED rather than left at its stored value. Without it, a write
+		// over an existing record lands on `Table.patch`, so there is no way to drop an attribute
+		// through the operations API — omitting it keeps the stored value and `null` stores a null
+		// (HarperFast/studio#1643).
+		//
+		// The flag is orthogonal to each operation's create rule, which is what makes one flag cover
+		// both useful shapes: `update` still requires an existing record (a missing one is skipped),
+		// while `upsert` still creates one — so `upsert` + `full_record` is exactly REST's
+		// `PUT /Table/id`, and `update` + `full_record` is that same replace, refused if the record
+		// isn't there. It has no effect on `insert`, which never writes over an existing record.
+		//
+		// Compared with the delete-then-insert this replaces, a full replace is one operation rather
+		// than two: the record is never absent between them, subscribers see a single write instead of
+		// a delete followed by an insert, and `__createdtime__` survives (`Table._writeUpdate` retains
+		// the stored created time on a full update and only stamps a new one for a new entry).
+		const fullRecord = upsertObj.full_record === true;
 		const Table = getDatabases()[upsertObj.schema][upsertObj.table];
 		const context: Context = {
 			user: upsertObj.hdb_user,
@@ -259,12 +277,24 @@ export class ResourceBridge extends BridgeMethods {
 						}
 					}
 				}
+				const unset = takeUnsetAttributes(record, Table.primaryKey);
+				// `__unset__` removes named attributes while everything else still merges, which a patch
+				// cannot express — so it resolves the merge here, against the record this transaction
+				// already loaded, and writes the result as a full replace. Doing it server-side inside the
+				// write transaction is the point: a client emulating this with read-then-replace races
+				// every other writer, and `full_record` would make the caller resend attributes it never
+				// meant to touch. Under `full_record` the submitted record is already the whole intended
+				// state, so there is nothing to merge and the names are simply dropped from it.
+				const toWrite = unset.length && existingRecord && !fullRecord ? { ...existingRecord, ...record } : record;
+				for (const attribute of unset) {
+					delete toWrite[attribute];
+				}
 				await (id == undefined
-					? Table.create(record, context)
-					: existingRecord
-						? Table.patch(record, context)
-						: Table.put(record, context));
-				keys.push(record[Table.primaryKey]);
+					? Table.create(toWrite, context)
+					: existingRecord && !fullRecord && !unset.length
+						? Table.patch(toWrite, context)
+						: Table.put(toWrite, context));
+				keys.push(toWrite[Table.primaryKey]);
 			}
 			return {
 				txn_time: (transaction as any).timestamp,
@@ -771,3 +801,23 @@ async function* groupRecordsInHistory(table, start?, end?, limit?) {
 	}
 	if (enqueued) yield enqueued;
 }
+
+/**
+ * Read and remove a record's `__unset__` list, returning the attribute names to drop.
+ *
+ * Removed from the record because it is a directive, not data: leaving it in place would store it as
+ * an attribute. Shape is already validated (`validation/insertValidator.ts`); the primary key is
+ * refused here instead, since only this layer knows which attribute that is. Unsetting it would
+ * otherwise be silently ignored — `Table._writeUpdate` re-asserts the primary key on a full update —
+ * and a directive that quietly does nothing is worse than one that says no.
+ */
+function takeUnsetAttributes(record: Record<string, unknown>, primaryKey: string): string[] {
+	const unset = record[OPERATIONS_UNSET_KEY];
+	if (unset === undefined) return [];
+	delete record[OPERATIONS_UNSET_KEY];
+	const names = unset as string[];
+	if (primaryKey && names.includes(primaryKey)) {
+		throw new ClientError(`'${primaryKey}' is the primary key of this table and cannot be unset`);
+	}
+	return names;
+}

dataLayer/harperBridge/bridgeUtility/insertUpdateValidate.js

@@ -4,6 +4,7 @@ const hdbUtils = require('../../../utility/common_utils.ts');
 const log = require('../../../utility/logging/harper_logger.ts');
 const { getDatabases } = require('../../../resources/databases.ts');
 const { ClientError } = require('../../../utility/errors/hdbError.ts');
+const { UNSET_ATTRIBUTES } = require('../../../utility/hdbTerms.ts');
 
 module.exports = insertUpdateValidate;
 
@@ -73,6 +74,10 @@ function insertUpdateValidate(writeObject) {
 		dups.add(hdbUtils.autoCast(record[hash_attribute]));
 
 		for (let attr in record) {
+			// `__unset__` names attributes to remove; it is not one itself. Collecting it would make an
+			// open (non-schemaDefined) table register `__unset__` as a real attribute, since
+			// upsertRecords feeds this list straight to Table.addAttributes.
+			if (attr === UNSET_ATTRIBUTES) continue;
 			attributes[attr] = 1;
 		}
 	});

dataLayer/insert.ts

@@ -15,6 +15,7 @@ import * as globalSchema from '../utility/globalSchema.ts';
 import log from '../utility/logging/harper_logger.ts';
 import { handleHDBError } from '../utility/errors/hdbError.ts';
 import { HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts';
+import * as terms from '../utility/hdbTerms.ts';
 
 const pGlobalSchema = util.promisify(globalSchema.getTableSchema);
 
@@ -89,6 +90,10 @@ export async function validation(writeObject: any) {
 		dups.add(hdbUtils.autoCast(record[hash_attribute]));
 
 		for (let attr in record) {
+			// Kept in step with the sync twin in harperBridge/bridgeUtility/insertUpdateValidate.js:
+			// `__unset__` names attributes to remove and is not one itself, so it must not be registered
+			// as a table attribute.
+			if (attr === terms.UNSET_ATTRIBUTES) continue;
 			attributes[attr] = 1;
 		}
 	});