Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 33 additions & 1 deletion src/main/scala/com/fulcrumgenomics/bam/Bams.scala
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,15 @@
import com.fulcrumgenomics.commons.io.Writer
import com.fulcrumgenomics.commons.util.LazyLogging
import com.fulcrumgenomics.fasta.ReferenceSequenceIterator
import com.fulcrumgenomics.util.{Io, ProgressLogger, Sorter}
import com.fulcrumgenomics.util.{Io, PrefixTrieSet, ProgressLogger, Sorter}
import htsjdk.samtools.SAMFileHeader.{GroupOrder, SortOrder}
import htsjdk.samtools.SamPairUtil.PairOrientation
import htsjdk.samtools._
import htsjdk.samtools.reference.ReferenceSequence
import htsjdk.samtools.util.{CloserUtil, CoordMath, Murmur3, SequenceUtil}

import java.io.Closeable
import scala.collection.mutable
import scala.math.{max, min}

/**
Expand Down Expand Up @@ -567,4 +568,35 @@
def close(): Unit = writer.close()
}
}

/** Builds a [[Writer]] of [[SamRecord]]s where all records for a given read name (e.g. pair, including any
* secondary and supplementary) are output to the writer. This is implemented via collecting the unique read
* names from records passed into `write()`, then upon closing, all records in the original BAM are examined and any
* records with a matching read name are written the provided writer.
*
* @param original the original BAM that will be filtered
* @param writer the final writer to which filtered records shuld be written
* @return
*/
def readNameFilteringWriter(original: PathToBam, writer: SamWriter): Writer[SamRecord] with Closeable = {
new Writer[SamRecord] with Closeable {
private val readNames = new PrefixTrieSet()
private val progress = new ProgressLogger(logger, noun="unique reads", verb="added", unit=1e6.toInt)

Check warning on line 584 in src/main/scala/com/fulcrumgenomics/bam/Bams.scala

View check run for this annotation

Codecov / codecov/patch

src/main/scala/com/fulcrumgenomics/bam/Bams.scala#L582-L584

Added lines #L582 - L584 were not covered by tests
override def write(rec: SamRecord): Unit = {
readNames.add(rec.name)
progress.record(rec)

Check warning on line 587 in src/main/scala/com/fulcrumgenomics/bam/Bams.scala

View check run for this annotation

Codecov / codecov/patch

src/main/scala/com/fulcrumgenomics/bam/Bams.scala#L586-L587

Added lines #L586 - L587 were not covered by tests
}
override def close(): Unit = {
val progress = new ProgressLogger(logger, verb="written", unit=1e6.toInt)
logger.info("Filtering original input based on read names")
val source = SamSource(original)
source.filter(rec => this.readNames.contains(rec.name)).foreach { rec =>
progress.record(rec)
writer.write(rec)

Check warning on line 595 in src/main/scala/com/fulcrumgenomics/bam/Bams.scala

View check run for this annotation

Codecov / codecov/patch

src/main/scala/com/fulcrumgenomics/bam/Bams.scala#L590-L595

Added lines #L590 - L595 were not covered by tests
}
source.close()
progress.logLast()

Check warning on line 598 in src/main/scala/com/fulcrumgenomics/bam/Bams.scala

View check run for this annotation

Codecov / codecov/patch

src/main/scala/com/fulcrumgenomics/bam/Bams.scala#L597-L598

Added lines #L597 - L598 were not covered by tests
}
}
}
}
82 changes: 82 additions & 0 deletions src/main/scala/com/fulcrumgenomics/util/PrefixTrieSet.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
/*
* The MIT License
*
* Copyright (c) 2025 Fulcrum Genomics
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*
*/
package com.fulcrumgenomics.util

class PrefixTrieSet(key: Option[Char] = None) {

private var nodes = new scala.collection.immutable.TreeMap[Char, PrefixTrieSet]
private var numSeen = 0L

def add(k: String): Long = {
if (k.isEmpty) {
this.numSeen += 1
this.numSeen
}
else {
val key = k.charAt(0)
val node = this.nodes.get(key) match {
case Some(node) => node
case None => {
val node = new PrefixTrieSet(Some(key))
this.nodes = this.nodes.updated(key, node)
node
}
}
node.add(k.substring(1))
}
}

def contains(k: String): Boolean = {
nodeFor(k).isDefined
}

def prefixes(): Iterator[String] = {
val thisIter = if (this.numSeen > 0) Iterator("") else Iterator.empty
thisIter ++ this.nodes.iterator.flatMap { case (char, node) =>
node.prefixes().map { suffix => f"${char}${suffix}" }
}
}

def values(): Iterator[(String, Long)] = {
val thisIter = if (this.numSeen > 0) Iterator(("", this.numSeen)) else Iterator.empty
thisIter ++ this.nodes.iterator.flatMap { case (char, node) =>
node.values().map { case (suffix, count) =>
(f"${char}${suffix}", count)
}
}
}

private def nodeFor(k: String): Option[PrefixTrieSet] = {
if (k.isEmpty) {
if (this.numSeen > 0) Some(this)
else None
}
else {
this.nodes.get(k.charAt(0)).flatMap { node =>
node.nodeFor(k.substring(1))
}
}
}
}
99 changes: 99 additions & 0 deletions src/test/scala/com/fulcrumgenomics/util/PrefixTrieSetTest.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
/*
* The MIT License
*
* Copyright (c) 2025 Fulcrum Genomics
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*
*/

package com.fulcrumgenomics.util

import com.fulcrumgenomics.testing.UnitSpec

import scala.util.Random

class PrefixTrieSetTest extends UnitSpec {

"PrefixTrieSet" should "support an empty string" in {
val set = new PrefixTrieSet()
set.contains("") shouldBe false
set.add("") shouldBe 1
set.contains("") shouldBe true
set.add("") shouldBe 2
set.prefixes().toSeq should contain theSameElementsAs Seq("")
}

it should "support strings with no common prefix" in {
val set = new PrefixTrieSet()
"ACGT".foreach { prefix =>
val key = prefix + "GATTACA"
set.contains(key) shouldBe false
set.add(key) shouldBe 1
set.contains(key) shouldBe true
set.add(key) shouldBe 2
set.contains(key) shouldBe true
}
set.prefixes().toSeq should contain theSameElementsAs "ACGT".map(c => c + "GATTACA")
}

it should "collapse prefixes" in {
val set = new PrefixTrieSet()
val keys = Seq(
"GATTACA",
"GATACA",
"CATTACA",
"CATTACA",
"GAT",
"",
"GATTACA"
)
keys.foreach(set.add)
keys.forall(set.contains) shouldBe true
set.prefixes().toSet should contain theSameElementsAs keys.toSet
set.values().toSeq should contain theSameElementsAs Seq(
("GATTACA", 2),
("GATACA", 1),
("CATTACA", 2),
("GAT", 1),
("", 1),
)
}

Seq(0, 1, 10, 100, 1000, 10000, 25000).foreach { numSequences =>
Seq(1, 2, 4, 8, 16, 32, 64).foreach { length =>
it should f"support adding in $numSequences random sequences of maximum length $length" in {
val bases = "ACGT"
val trieSet = new PrefixTrieSet()
val hashSet = new scala.collection.mutable.HashSet[String]()
val minLength = length / 2
Range.inclusive(start=1, end=numSequences).foreach { _ =>
val keyLength = Random.between(minLength, length)
val key: String = Range.inclusive(start=1, end=keyLength).map { _ =>
bases.charAt(Random.nextInt(4))
}.toString
trieSet.add(key)
hashSet.add(key)
}
trieSet.prefixes().toSeq should contain theSameElementsAs hashSet.toSeq
trieSet.values().map(_._2).sum shouldBe numSequences
}
}
}
}