Skip to content

Commit 8f2d1f6

Browse files
icewind1991icewind1991
committed
store unencrypted size in the unencrypted_size column
Signed-off-by: Robin Appelman <robin@icewind.nl>
1 parent 8342964 commit 8f2d1f6

9 files changed

Lines changed: 133 additions & 52 deletions

File tree

‎lib/private/Files/Cache/Cache.php‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -428,7 +428,7 @@ public function update($id, array $data) {
428428
protected function normalizeData(array $data): array {
429429
$fields = [
430430
'path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'storage_mtime', 'encrypted',
431-
'etag', 'permissions', 'checksum', 'storage'];
431+
'etag', 'permissions', 'checksum', 'storage', 'unencrypted_size'];
432432
$extensionFields = ['metadata_etag', 'creation_time', 'upload_time'];
433433

434434
$doNotCopyStorageMTime = false;
@@ -873,8 +873,10 @@ public function calculateFolderSize($path, $entry = null) {
873873
$id = $entry['fileid'];
874874

875875
$query = $this->getQueryBuilder();
876-
$query->selectAlias($query->func()->sum('size'), 'f1')
877-
->selectAlias($query->func()->min('size'), 'f2')
876+
$query->selectAlias($query->func()->sum('size'), 'size_sum')
877+
->selectAlias($query->func()->min('size'), 'size_min')
878+
->selectAlias($query->func()->sum('unencrypted_size'), 'unencrypted_sum')
879+
->selectAlias($query->func()->min('unencrypted_size'), 'unencrypted_min')
878880
->from('filecache')
879881
->whereStorageId($this->getNumericStorageId())
880882
->whereParent($id);
@@ -884,16 +886,30 @@ public function calculateFolderSize($path, $entry = null) {
884886
$result->closeCursor();
885887

886888
if ($row) {
887-
[$sum, $min] = array_values($row);
889+
['size_sum' => $sum, 'size_min' => $min, 'unencrypted_sum' => $unencryptedSum, 'unencrypted_min' => $unencryptedMin] = $row;
888890
$sum = 0 + $sum;
889891
$min = 0 + $min;
890892
if ($min === -1) {
891893
$totalSize = $min;
892894
} else {
893895
$totalSize = $sum;
894896
}
897+
if ($unencryptedMin === -1 || $min === -1) {
898+
$unencryptedTotal = $unencryptedMin;
899+
} else {
900+
$unencryptedTotal = $unencryptedSum;
901+
}
895902
if ($entry['size'] !== $totalSize) {
896-
$this->update($id, ['size' => $totalSize]);
903+
if ($unencryptedTotal !== null) {
904+
$this->update($id, [
905+
'size' => $totalSize,
906+
'unencrypted_size' => $unencryptedTotal,
907+
]);
908+
} else {
909+
$this->update($id, [
910+
'size' => $totalSize
911+
]);
912+
}
897913
}
898914
}
899915
}

‎lib/private/Files/Cache/CacheEntry.php‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,4 +132,12 @@ public function getData() {
132132
public function __clone() {
133133
$this->data = array_merge([], $this->data);
134134
}
135+
136+
public function getUnencryptedSize(): int {
137+
if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) {
138+
return $this->data['unencrypted_size'];
139+
} else {
140+
return $this->data['size'];
141+
}
142+
}
135143
}

‎lib/private/Files/Cache/CacheQueryBuilder.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public function __construct(IDBConnection $connection, SystemConfig $systemConfi
4444
public function selectFileCache(string $alias = null) {
4545
$name = $alias ? $alias : 'filecache';
4646
$this->select("$name.fileid", 'storage', 'path', 'path_hash', "$name.parent", 'name', 'mimetype', 'mimepart', 'size', 'mtime',
47-
'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'metadata_etag', 'creation_time', 'upload_time')
47+
'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'metadata_etag', 'creation_time', 'upload_time', 'unencrypted_size')
4848
->from('filecache', $name)
4949
->leftJoin($name, 'filecache_extended', 'fe', $this->expr()->eq("$name.fileid", 'fe.fileid'));
5050

‎lib/private/Files/Cache/Propagator.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,10 @@ public function propagateChange($internalPath, $time, $sizeDifference = 0) {
109109
$builder->func()->add('size', $builder->createNamedParameter($sizeDifference)),
110110
$builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT)
111111
))
112+
->set('unencrypted_size', $builder->func()->greatest(
113+
$builder->func()->add('unencrypted_size', $builder->createNamedParameter($sizeDifference)),
114+
$builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT)
115+
))
112116
->where($builder->expr()->eq('storage', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
113117
->andWhere($builder->expr()->in('path_hash', $hashParams))
114118
->andWhere($builder->expr()->gt('size', $builder->expr()->literal(-1, IQueryBuilder::PARAM_INT)));

‎lib/private/Files/FileInfo.php‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,11 @@ public function __construct($path, $storage, $internalPath, $data, $mount, $owne
101101
$this->data = $data;
102102
$this->mount = $mount;
103103
$this->owner = $owner;
104-
$this->rawSize = $this->data['size'] ?? 0;
104+
if (isset($this->data['unencrypted_size'])) {
105+
$this->rawSize = $this->data['unencrypted_size'];
106+
} else {
107+
$this->rawSize = $this->data['size'] ?? 0;
108+
}
105109
}
106110

107111
public function offsetSet($offset, $value): void {
@@ -208,7 +212,12 @@ public function getEtag() {
208212
public function getSize($includeMounts = true) {
209213
if ($includeMounts) {
210214
$this->updateEntryfromSubMounts();
211-
return isset($this->data['size']) ? 0 + $this->data['size'] : 0;
215+
216+
if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) {
217+
return $this->data['unencrypted_size'];
218+
} else {
219+
return isset($this->data['size']) ? 0 + $this->data['size'] : 0;
220+
}
212221
} else {
213222
return $this->rawSize;
214223
}
@@ -386,7 +395,19 @@ private function updateEntryfromSubMounts() {
386395
* @param string $entryPath full path of the child entry
387396
*/
388397
public function addSubEntry($data, $entryPath) {
389-
$this->data['size'] += isset($data['size']) ? $data['size'] : 0;
398+
if (!$data) {
399+
return;
400+
}
401+
$hasUnencryptedSize = isset($data['unencrypted_size']) && $data['unencrypted_size'] > 0;
402+
if ($hasUnencryptedSize) {
403+
$subSize = $data['unencrypted_size'];
404+
} else {
405+
$subSize = $data['size'] ?: 0;
406+
}
407+
$this->data['size'] += $subSize;
408+
if ($hasUnencryptedSize) {
409+
$this->data['unencrypted_size'] += $subSize;
410+
}
390411
if (isset($data['mtime'])) {
391412
$this->data['mtime'] = max($this->data['mtime'], $data['mtime']);
392413
}

‎lib/private/Files/Storage/Wrapper/Encryption.php‎

Lines changed: 58 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
* along with this program. If not, see <http://www.gnu.org/licenses/>
3434
*
3535
*/
36+
3637
namespace OC\Files\Storage\Wrapper;
3738

3839
use OC\Encryption\Exceptions\ModuleDoesNotExistsException;
@@ -41,6 +42,7 @@
4142
use OC\Files\Cache\CacheEntry;
4243
use OC\Files\Filesystem;
4344
use OC\Files\Mount\Manager;
45+
use OC\Files\ObjectStore\ObjectStoreStorage;
4446
use OC\Files\Storage\LocalTempFileTrait;
4547
use OC\Memcache\ArrayCache;
4648
use OCP\Encryption\Exceptions\GenericEncryptionException;
@@ -139,28 +141,36 @@ public function filesize($path) {
139141
$size = $this->unencryptedSize[$fullPath];
140142
// update file cache
141143
if ($info instanceof ICacheEntry) {
142-
$info = $info->getData();
143144
$info['encrypted'] = $info['encryptedVersion'];
144145
} else {
145146
if (!is_array($info)) {
146147
$info = [];
147148
}
148149
$info['encrypted'] = true;
150+
$info = new CacheEntry($info);
149151
}
150152

151-
$info['size'] = $size;
152-
$this->getCache()->put($path, $info);
153+
if ($size !== $info->getUnencryptedSize()) {
154+
$this->getCache()->update($info->getId(), [
155+
'unencrypted_size' => $size
156+
]);
157+
}
153158

154159
return $size;
155160
}
156161

157162
if (isset($info['fileid']) && $info['encrypted']) {
158-
return $this->verifyUnencryptedSize($path, $info['size']);
163+
return $this->verifyUnencryptedSize($path, $info->getUnencryptedSize());
159164
}
160165

161166
return $this->storage->filesize($path);
162167
}
163168

169+
/**
170+
* @param string $path
171+
* @param array $data
172+
* @return array
173+
*/
164174
private function modifyMetaData(string $path, array $data): array {
165175
$fullPath = $this->getFullPath($path);
166176
$info = $this->getCache()->get($path);
@@ -170,7 +180,7 @@ private function modifyMetaData(string $path, array $data): array {
170180
$data['size'] = $this->unencryptedSize[$fullPath];
171181
} else {
172182
if (isset($info['fileid']) && $info['encrypted']) {
173-
$data['size'] = $this->verifyUnencryptedSize($path, $info['size']);
183+
$data['size'] = $this->verifyUnencryptedSize($path, $info->getUnencryptedSize());
174184
$data['encrypted'] = true;
175185
}
176186
}
@@ -478,7 +488,7 @@ public function fopen($path, $mode) {
478488
*
479489
* @return int unencrypted size
480490
*/
481-
protected function verifyUnencryptedSize($path, $unencryptedSize) {
491+
protected function verifyUnencryptedSize(string $path, int $unencryptedSize): int {
482492
$size = $this->storage->filesize($path);
483493
$result = $unencryptedSize;
484494

@@ -510,7 +520,7 @@ protected function verifyUnencryptedSize($path, $unencryptedSize) {
510520
*
511521
* @return int calculated unencrypted size
512522
*/
513-
protected function fixUnencryptedSize($path, $size, $unencryptedSize) {
523+
protected function fixUnencryptedSize(string $path, int $size, int $unencryptedSize): int {
514524
$headerSize = $this->getHeaderSize($path);
515525
$header = $this->getHeader($path);
516526
$encryptionModule = $this->getEncryptionModule($path);
@@ -581,7 +591,9 @@ protected function fixUnencryptedSize($path, $size, $unencryptedSize) {
581591
$cache = $this->storage->getCache();
582592
if ($cache) {
583593
$entry = $cache->get($path);
584-
$cache->update($entry['fileid'], ['size' => $newUnencryptedSize]);
594+
$cache->update($entry['fileid'], [
595+
'unencrypted_size' => $newUnencryptedSize
596+
]);
585597
}
586598

587599
return $newUnencryptedSize;
@@ -621,7 +633,12 @@ private function fread_block($handle, int $blockSize): string {
621633
* @param bool $preserveMtime
622634
* @return bool
623635
*/
624-
public function moveFromStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime = true) {
636+
public function moveFromStorage(
637+
Storage\IStorage $sourceStorage,
638+
$sourceInternalPath,
639+
$targetInternalPath,
640+
$preserveMtime = true
641+
) {
625642
if ($sourceStorage === $this) {
626643
return $this->rename($sourceInternalPath, $targetInternalPath);
627644
}
@@ -656,7 +673,13 @@ public function moveFromStorage(Storage\IStorage $sourceStorage, $sourceInternal
656673
* @param bool $isRename
657674
* @return bool
658675
*/
659-
public function copyFromStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime = false, $isRename = false) {
676+
public function copyFromStorage(
677+
Storage\IStorage $sourceStorage,
678+
$sourceInternalPath,
679+
$targetInternalPath,
680+
$preserveMtime = false,
681+
$isRename = false
682+
) {
660683

661684
// TODO clean this up once the underlying moveFromStorage in OC\Files\Storage\Wrapper\Common is fixed:
662685
// - call $this->storage->copyFromStorage() instead of $this->copyBetweenStorage
@@ -676,7 +699,13 @@ public function copyFromStorage(Storage\IStorage $sourceStorage, $sourceInternal
676699
* @param bool $isRename
677700
* @param bool $keepEncryptionVersion
678701
*/
679-
private function updateEncryptedVersion(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, $keepEncryptionVersion) {
702+
private function updateEncryptedVersion(
703+
Storage\IStorage $sourceStorage,
704+
$sourceInternalPath,
705+
$targetInternalPath,
706+
$isRename,
707+
$keepEncryptionVersion
708+
) {
680709
$isEncrypted = $this->encryptionManager->isEnabled() && $this->shouldEncrypt($targetInternalPath);
681710
$cacheInformation = [
682711
'encrypted' => $isEncrypted,
@@ -725,7 +754,13 @@ private function updateEncryptedVersion(Storage\IStorage $sourceStorage, $source
725754
* @return bool
726755
* @throws \Exception
727756
*/
728-
private function copyBetweenStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime, $isRename) {
757+
private function copyBetweenStorage(
758+
Storage\IStorage $sourceStorage,
759+
$sourceInternalPath,
760+
$targetInternalPath,
761+
$preserveMtime,
762+
$isRename
763+
) {
729764

730765
// for versions we have nothing to do, because versions should always use the
731766
// key from the original file. Just create a 1:1 copy and done
@@ -743,7 +778,7 @@ private function copyBetweenStorage(Storage\IStorage $sourceStorage, $sourceInte
743778
if (isset($info['encrypted']) && $info['encrypted'] === true) {
744779
$this->updateUnencryptedSize(
745780
$this->getFullPath($targetInternalPath),
746-
$info['size']
781+
$info->getUnencryptedSize()
747782
);
748783
}
749784
$this->updateEncryptedVersion($sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, true);
@@ -808,13 +843,6 @@ private function copyBetweenStorage(Storage\IStorage $sourceStorage, $sourceInte
808843
return (bool)$result;
809844
}
810845

811-
/**
812-
* get the path to a local version of the file.
813-
* The local version of the file can be temporary and doesn't have to be persistent across requests
814-
*
815-
* @param string $path
816-
* @return string
817-
*/
818846
public function getLocalFile($path) {
819847
if ($this->encryptionManager->isEnabled()) {
820848
$cachedFile = $this->getCachedFile($path);
@@ -825,42 +853,25 @@ public function getLocalFile($path) {
825853
return $this->storage->getLocalFile($path);
826854
}
827855

828-
/**
829-
* Returns the wrapped storage's value for isLocal()
830-
*
831-
* @return bool wrapped storage's isLocal() value
832-
*/
833856
public function isLocal() {
834857
if ($this->encryptionManager->isEnabled()) {
835858
return false;
836859
}
837860
return $this->storage->isLocal();
838861
}
839862

840-
/**
841-
* see https://www.php.net/manual/en/function.stat.php
842-
* only the following keys are required in the result: size and mtime
843-
*
844-
* @param string $path
845-
* @return array
846-
*/
847863
public function stat($path) {
848864
$stat = $this->storage->stat($path);
865+
if (!$stat) {
866+
return false;
867+
}
849868
$fileSize = $this->filesize($path);
850869
$stat['size'] = $fileSize;
851870
$stat[7] = $fileSize;
852871
$stat['hasHeader'] = $this->getHeaderSize($path) > 0;
853872
return $stat;
854873
}
855874

856-
/**
857-
* see https://www.php.net/manual/en/function.hash.php
858-
*
859-
* @param string $type
860-
* @param string $path
861-
* @param bool $raw
862-
* @return string
863-
*/
864875
public function hash($type, $path, $raw = false) {
865876
$fh = $this->fopen($path, 'rb');
866877
$ctx = hash_init($type);
@@ -1068,6 +1079,13 @@ public function writeStream(string $path, $stream, int $size = null): int {
10681079
[$count, $result] = \OC_Helper::streamCopy($stream, $target);
10691080
fclose($stream);
10701081
fclose($target);
1082+
1083+
// object store, stores the size after write and doesn't update this during scan
1084+
// manually store the unencrypted size
1085+
if ($result && $this->getWrapperStorage()->instanceOfStorage(ObjectStoreStorage::class)) {
1086+
$this->getCache()->put($path, ['unencrypted_size' => $count]);
1087+
}
1088+
10711089
return $count;
10721090
}
10731091
}

‎lib/public/Files/Cache/ICacheEntry.php‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,4 +162,14 @@ public function getCreationTime(): ?int;
162162
* @since 18.0.0
163163
*/
164164
public function getUploadTime(): ?int;
165+
166+
/**
167+
* Get the unencrypted size
168+
*
169+
* This might be different from the result of getSize
170+
*
171+
* @return int
172+
* @since 25.0.0
173+
*/
174+
public function getUnencryptedSize(): int;
165175
}

0 commit comments

Comments
 (0)