Akka-CQRS（8）- CQRS Reader Actor 应用实例
前面我们已经讨论了CQRS-Reader-Actor的基本工作原理，现在是时候在之前那个POS例子里进行实际的应用示范了。
假如我们有个业务系统也是在cassandra上的，那么reader就需要把从日志读出来的事件恢复成cassandra表里的数据行row。首先，我们需要在cassandra上创建相关的keyspace和table。下面是在scala中使用cassandra-java-driver的例子：
import com.datastax.driver.core._
import akka.actor.ActorSystem
import akka.stream.ActorMaterializer
import sdp.cql.engine._
import CQLEngine._
import CQLHelpers._
import monix.execution.Scheduler.Implicits.global
import scala.util._ object CQLCreatTables extends App {
//#init-mat
implicit val cqlsys = ActorSystem("cqlSystem")
implicit val mat = ActorMaterializer()
// implicit val ec = cqlsys.dispatcher val cluster = new Cluster
.Builder()
.addContactPoints("192.168.11.189")
.withPort(9042)
.build() useJava8DateTime(cluster)
implicit val session = cluster.connect() val createKeyspace = """
CREATE KEYSPACE pos_on_cloud WITH replication = { //pos业务数据库
'class': 'SimpleStrategy',
'replication_factor': '3'
}""" val createVchLog ="""
CREATE TABLE pos_on_cloud.vch_log ( //销售单号日志 （可以从某日开始重新运算交易）
terminal text,
txndate text,
vchnum int,
begin_seq bigint,
end_seq bigint,
PRIMARY KEY (terminal,txndate,vchnum)
)""" val createTxnItems ="""
CREATE TABLE pos_on_cloud.txn_log ( //pos交易记录表
terminal text,
txndate text,
txntime text,
opr text,
num int,
seq int,
txntype int,
salestype int,
qty int,
price int,
amount int,
disc int,
dscamt int,
member text,
code text,
acct text,
dpt text,
PRIMARY KEY (terminal,txndate,num,seq)
)""" val createTxnSuspend ="""
CREATE TABLE pos_on_cloud.txn_hold ( //临时挂单表
terminal text,
txndate text,
txntime text,
opr text,
num int,
seq int,
txntype int,
salestype int,
qty int,
price int,
amount int,
disc int
dscamt int,
member text,
code text,
acct text,
dpt text,
PRIMARY KEY (terminal,txndate,num,seq)
)""" val ctxKeyspace = CQLContext().setCommand(createKeyspace)
val ctxVchlog = CQLContext().setCommand(createVchLog)
val ctxTxnlog = CQLContext().setCommand(createTxnItems)
val ctxTxnhold = CQLContext().setCommand(createTxnSuspend) val results = for {
stsKeyspace <- cqlExecute(ctxKeyspace)
stsVchlog <- cqlExecute(ctxVchlog)
stsTxnlog <- cqlExecute(ctxTxnlog)
stsTxnhold <- cqlExecute(ctxTxnhold)
} yield (stsKeyspace,stsVchlog,stsTxnlog,stsTxnhold) val task = results.value.value val cancellableFut = task.runToFuture
cancellableFut.onComplete {
case Success(value) =>
println(s"returned status: $value")
case Failure(ex) =>
System.err.println(s"ERROR: ${ex.getMessage}") } // cancellableFut.cancel()
/*
val cancelable = task.runAsync { result =>
result match {
case Right(value) =>
println(value)
case Left(ex) =>
System.err.println(s"ERROR: ${ex.getMessage}")
}
} */ scala.io.StdIn.readLine() session.close()
cluster.close() cqlsys.terminate() }
这里面调用了之前PICE系列博文中设计的CassandraEngine里的工具源代码。下面是用CassandraEngine工具向cassandra表里插入数据的示范代码：
object DBWriter {
def writeTxnsToDB(vchnum: Int, susp: Boolean, txns: List[TxnItem])(pid: String, bseq: Long, eseq: Long) = {
import monix.execution.Scheduler.Implicits.global
val cluster = new Cluster
.Builder()
.addContactPoints("192.168.11.189")
.withPort(9042)
.build()
useJava8DateTime(cluster)
implicit val session = cluster.connect()
val insertVchLog = """
|insert into pos_on_cloud.vch_log(terminal,txndate,vchnum,begin_seq,end_seq)
|values(?,?,?,?,?)
|""".stripMargin
val insertTxns = """
|insert into pos_on_cloud.txn_log(terminal,txndate,txntime,opr,num,seq,txntype,salestype,
|qty,price,amount,disc,dscamt,member,code,acct,dpt)
|values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
""".stripMargin
val insertSusp = """
|insert into pos_on_cloud.txn_hold(terminal,txndate,txntime,opr,num,seq,txntype,salestype,
|qty,price,amount,disc,dscamt,member,code,acct,dpt)
|values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
""".stripMargin
val vchParams: Seq[Object] = Seq(
pid.asInstanceOf[Object],
LocalDate.now.format(DateTimeFormatter.ofPattern("yyyyMMdd")).asInstanceOf[Object],
vchnum.asInstanceOf[Object],
bseq.asInstanceOf[Object],
eseq.asInstanceOf[Object]
)
val txnParams: Seq[Seq[Object]] = txns.foldRight(Seq[Seq[Object]]()) { (txnitem,b) =>
(Seq(pid.asInstanceOf[Object]) ++ ccToList(txnitem)) +: b
}
val ctxVchlog = CQLContext().setCommand(insertVchLog, 1,vchParams)
val ctxTxnlog = CQLContext().setCommand((if(susp) insertSusp else insertTxns),txnParams.size,txnParams)
val results = for {
stsVchlog <- cqlExecute(ctxVchlog)
stsTxnlog <- cqlExecute(ctxTxnlog)
} yield (stsTxnlog)
val task = results.value.value
val cancellableFut = task.runToFuture
cancellableFut.onComplete {
case Success(value) =>
println(s"returned status: $value")
session.close()
cluster.close()
case Failure(ex) =>
System.err.println(s"ERROR: ${ex.getMessage}")
session.close()
cluster.close()
}
// cqlsys.terminate()
}
def getMapFromCC(cc: Product) = cc.getClass.getDeclaredFields.map( _.getName ) // all field names
.zip( cc.productIterator.to ).toMap // zipped with all values
def ccFieldsToMap(cc: Product) = {
val values = cc.productIterator
cc.getClass.getDeclaredFields.map( _.getName -> (values.next).asInstanceOf[Object] ).toMap
}
def ccToList(cc: Product) = {
val values = cc.productIterator
cc.getClass.getDeclaredFields.map(_ => (values.next).asInstanceOf[Object] ).toList
}
def ccToMap(cc: Product): Map[String, Object] = {
val values = cc.productIterator
cc.getClass.getDeclaredFields.map {
_.getName -> (values.next() match {
case p: Product if p.productArity > 0 => ccToMap(p)
case x => x.asInstanceOf[Object]
})
}.toMap
}
}
用cqlsh: select * from txn_hold 检查了一下，插入数据正确。对于这种批量数据同类处理，可能用akka-stream会更方便高效：
val actionStreamVs = CassandraActionStream(insertVchLog,vsToParams)
.setParallelism(2)
.setProcessOrder(false)
val actionFlowVs: Flow[Seq[Object],Seq[Object],NotUsed] = actionStreamVs.performOnRow val sinkVs = Sink.foreach[Seq[Object]]{ r =>
log.step(s"insert: $insertVchLog, values: ${r}")
}
// insert to vch_log
val stsVs = Source.fromIterator(() => Seq(vchParams).iterator).via(actionFlowVs).to(sinkVs).run() val insertTxn = if (susp) insertSusp else insertTxns val txnitemToParams: TxnItem => Seq[Object] = txn =>
(Seq(pid.asInstanceOf[Object]) ++ ccToList(txn)) val actionStreamTxn = CassandraActionStream(insertTxn,txnitemToParams)
.setParallelism(2)
.setProcessOrder(false)
val actionFlowTxn: Flow[TxnItem,TxnItem,NotUsed] = actionStreamTxn.performOnRow val sinkTxn = Sink.foreach[TxnItem]{ r =>
log.step(s"insert: $insertTxn, values: ${r}")
}
// insert to txn_???
val stsTxn = Source.fromIterator(() => txns.iterator).via(actionFlowTxn).to(sinkTxn).run()
检查cassandra数据库表，结果正确。用stream方式来做重复类型的处理会比较方便，在当前这个例子的场合下建议使用。
好了，完成了事件日志读取和转换成数据行格式并写入数据库表后，下一步就是建一个reader-actor负责完成这一轮工作。这个reader-actor只根据下面这个消息进行相关的工作：
case class PerformRead(pid: String, vchnum: Int, bseq: Long, eseq: Long)
这个消息描述了读端需要读取的日志记录范围和persistenceId。然后再加一个远程路由remote-router，负责按照某种算法来向各个集群节点上的reader-actor分发读端任务。下面是reader-actor：
package datatech.cloud.pos
import akka.actor._
import akka.cluster._
import akka.pattern._
import scala.concurrent.duration._
import com.typesafe.config.ConfigFactory
import sdp.logging.LogSupport
import Messages._
import Reader._ object ActionReader {
def actionReaderProps(trace: Boolean): Props = Props(new ActionReader(trace)) //backoff suppervisor must use onStop mode
//respond only to failure of child
def readerProps(trace:Boolean): Props = {
val options = BackoffOpts.onFailure(
childProps = actionReaderProps(trace),
childName = "cqrs-reader",
minBackoff = 1 second,
maxBackoff = 10 seconds,
randomFactor = 0.20
).withMaxNrOfRetries(3)
BackoffSupervisor.props(options)
} def create(port: Int): Unit = {
val config = ConfigFactory.parseString(s"akka.remote.netty.tcp.port=$port")
.withFallback(ConfigFactory.load()) val system = ActorSystem("cloud-pos-server", config)
system.actorOf(readerProps(true),"reader")
}
} class ActionReader(trace: Boolean) extends Actor with LogSupport {
val cluster = Cluster(context.system)
val host = Cluster(context.system).selfAddress.host.get
implicit val nodeAddress: NodeAddress = NodeAddress(cluster.selfAddress.toString)
val readerId = "ActionReader"
log.stepOn = trace log.step(s"${nodeAddress.address}-${readerId}") override def preRestart(reason: Throwable, message: Option[Any]): Unit = {
super.preRestart(reason, message)
log.step(s"${nodeAddress.address}-${readerId} Restarting for $message ...")
} override def postRestart(reason: Throwable): Unit = {
super.postRestart(reason)
log.step(s"${nodeAddress.address}-${readerId} restarted for ${reason.getMessage}.")
} override def postStop(): Unit = {
log.step(s"${nodeAddress.address}-${readerId} stooped.")
} override def preStart(): Unit = {
log.step(s"${nodeAddress.address}-${readerId} Starting ...")
} var debugConfig: com.typesafe.config.Config = _
var debug: Boolean = _
try { debugConfig = ConfigFactory.load("pos.conf").getConfig("pos.server")
debug = debugConfig.getBoolean("debug")
}
catch {
case _ : Throwable => debug = false
} log.step(s"${nodeAddress.address}-${readerId} debug mode = $debug") implicit val debugMode = DebugMode(debug) override def receive: Receive = {
case PerformRead(pid, vchnum, bseq, eseq) =>
log.step(s"${nodeAddress.address}-${readerId} PerformRead($pid, $vchnum, $bseq, $eseq)")
readActions(host,bseq,eseq,pid,vchnum)(context.system,context.dispatcher,nodeAddress)
case msg @ _ =>
log.step(s"${nodeAddress.address}-${readerId} receive unsupported command:[$msg]")
} }
这是一个在backoffSupervisor后面的actor。remote-router是从配置文件定义创建的：
akka.actor.deployment {
/readerRouter/readerRouter = {
# Router type provided by metrics extension.
router = cluster-metrics-adaptive-group
# Router parameter specific for metrics extension.
# metrics-selector = heap
# metrics-selector = load
# metrics-selector = cpu
metrics-selector = mix
#
routees.paths = ["/user/reader"]
cluster {
max-nr-of-instances-per-node = 10
max-total-nr-of-instances = 1000
enabled = on
allow-local-routees = on
}
}
}
ReaderRouter的代码如下：
package datatech.cloud.pos
import akka.actor._
import akka.routing._
import akka.cluster._
import com.typesafe.config.ConfigFactory class ReaderRouter extends Actor {
val router = context.actorOf(FromConfig.props(), name = "readerRouter") def receive: Receive = {
case msg => router ! msg
}
} object ReaderRouter {
var router: ActorRef = _
def props = Props(new ReaderRouter) def create(port: Int) = {
val config = ConfigFactory.parseString(s"akka.remote.netty.tcp.port=$port")
.withFallback(ConfigFactory.load()) val system = ActorSystem("cloud-pos-server",config) Cluster(system).registerOnMemberUp{
router = system.actorOf(props,"readerRouter")
} } def getRouter = router
}
好了，可以写个例子来测试运行这个router/routee。reader-actor所做的工作在前面的讨论里已经测试过了。
ackage datatech.cloud.pos
import akka.actor._ import datatech.cloud.pos.Messages.PerformRead
object ReaderDemo extends App {
ActionReader.create(2551) ActionReader.create(2552) ActionReader.create(2553) ReaderRouter.create(2558) scala.io.StdIn.readLine() val router = ReaderRouter.getRouter router ! PerformRead("1022",111,0,Long.MaxValue) scala.io.StdIn.readLine() router ! PerformRead("1022",222,0,Long.MaxValue) scala.io.StdIn.readLine() }
在这个例子里我们先在本机的2551,2552,2553端口上部署了routees, 即reader-actor。然后在2558端口部署router，再向router发送任务PerformRead。这里有些东西值得留意：akka-cluster使用了netty，而netty也需要占用一个端口。在配置文件里：
remote {
log-remote-lifecycle-events = on
netty.tcp {
hostname = "192.168.11.189"
# port set to 0 for netty to randomly choose from
port = 0
}
}
下面是本次示范的源代码：
project/plugin.sbt
addSbtPlugin("com.lightbend.akka.grpc" % "sbt-akka-grpc" % "0.6.1")
addSbtPlugin("com.lightbend.sbt" % "sbt-javaagent" % "0.1.4") // ALPN agent
build.sbt
name := "akka-cluster-reader" version := "0.1" scalaVersion := "2.12.8" scalacOptions += "-Ypartial-unification" // in build.sbt:
//enablePlugins(AkkaGrpcPlugin)
// ALPN agent
//enablePlugins(JavaAgent)
//javaAgents += "org.mortbay.jetty.alpn" % "jetty-alpn-agent" % "2.0.9" % "runtime;test" libraryDependencies := Seq(
"com.typesafe.akka" %% "akka-cluster-metrics" % "2.5.19",
"com.typesafe.akka" %% "akka-cluster-sharding" % "2.5.19",
"com.typesafe.akka" %% "akka-persistence" % "2.5.19",
"com.lightbend.akka" %% "akka-stream-alpakka-cassandra" % "1.0.1",
"org.mongodb.scala" %% "mongo-scala-driver" % "2.6.0",
"com.lightbend.akka" %% "akka-stream-alpakka-mongodb" % "1.0.1",
"com.typesafe.akka" %% "akka-persistence-query" % "2.5.19",
"com.typesafe.akka" %% "akka-persistence-cassandra" % "0.93",
"com.typesafe.akka" %% "akka-persistence-cassandra-launcher" % "0.93" % Test,
"com.datastax.cassandra" % "cassandra-driver-core" % "3.6.0",
"com.datastax.cassandra" % "cassandra-driver-extras" % "3.6.0",
"ch.qos.logback" % "logback-classic" % "1.2.3",
"io.monix" %% "monix" % "3.0.0-RC2",
"org.typelevel" %% "cats-core" % "2.0.0-M1",
"com.thesamet.scalapb" %% "scalapb-runtime" % scalapb.compiler.Version.scalapbVersion % "protobuf",
"com.thesamet.scalapb" %% "scalapb-runtime-grpc" % scalapb.compiler.Version.scalapbVersion ) // (optional) If you need scalapb/scalapb.proto or anything from
// google/protobuf/*.proto
//libraryDependencies += "com.thesamet.scalapb" %% "scalapb-runtime" % scalapb.compiler.Version.scalapbVersion % "protobuf" PB.targets in Compile := Seq(
scalapb.gen() -> (sourceManaged in Compile).value
)
resources/application.conf
akka.actor.warn-about-java-serializer-usage = off
akka.log-dead-letters-during-shutdown = off
akka.log-dead-letters = off
akka.remote.use-passive-connections=off akka {
loglevel = INFO
actor {
provider = "cluster"
} remote {
log-remote-lifecycle-events = on
netty.tcp {
hostname = "192.168.11.189"
# port set to 0 for netty to randomly choose from
port = 0
}
} cluster {
seed-nodes = [
"akka.tcp://cloud-pos-server@192.168.11.189:2551"]
log-info = off
sharding {
role = "shard"
passivate-idle-entity-after = 10 m
}
} persistence {
journal.plugin = "cassandra-journal"
snapshot-store.plugin = "cassandra-snapshot-store"
} } cassandra-journal {
contact-points = ["192.168.11.189"]
} cassandra-snapshot-store {
contact-points = ["192.168.11.189"]
} # Enable metrics extension in akka-cluster-metrics.
akka.extensions=["akka.cluster.metrics.ClusterMetricsExtension"] akka.actor.deployment {
/readerRouter/readerRouter = {
# Router type provided by metrics extension.
router = cluster-metrics-adaptive-group
# Router parameter specific for metrics extension.
# metrics-selector = heap
# metrics-selector = load
# metrics-selector = cpu
metrics-selector = mix
#
routees.paths = ["/user/reader"]
cluster {
max-nr-of-instances-per-node = 10
max-total-nr-of-instances = 1000
enabled = on
#default in reference.conf
#allow-local-routees = on
#very important to set this off, could cause missing msg in local cluster
allow-local-routees = off
}
}
} dbwork-dispatcher {
# Dispatcher is the name of the event-based dispatcher
type = Dispatcher
# What kind of ExecutionService to use
executor = "fork-join-executor"
# Configuration for the fork join pool
fork-join-executor {
# Min number of threads to cap factor-based parallelism number to
parallelism-min = 2
# Parallelism (threads) ... ceil(available processors * factor)
parallelism-factor = 2.0
# Max number of threads to cap factor-based parallelism number to
parallelism-max = 10
}
# Throughput defines the maximum number of messages to be
# processed per actor before the thread jumps to the next actor.
# Set to 1 for as fair as possible.
throughput = 100
}
logback.xml
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<Pattern>
%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n
</Pattern>
</encoder>
</appender> <logger name="datatech" level="info"
additivity="false">
<appender-ref ref="STDOUT" />
</logger> <logger name="sdp" level="info"
additivity="false">
<appender-ref ref="STDOUT" />
</logger> <root level="warn">
<appender-ref ref="STDOUT" />
</root>
</configuration>
ActionReader.scala
package datatech.cloud.pos
import akka.actor._
import akka.cluster._
import akka.pattern._
import scala.concurrent.duration._
import com.typesafe.config.ConfigFactory
import sdp.logging.LogSupport
import Messages._
import Reader._ object ActionReader {
def actionReaderProps(trace: Boolean): Props = Props(new ActionReader(trace)) //backoff suppervisor must use onStop mode
//respond only to failure of child
def readerProps(trace:Boolean): Props = {
val options = BackoffOpts.onFailure(
childProps = actionReaderProps(trace),
childName = "cqrs-reader",
minBackoff = 1 second,
maxBackoff = 10 seconds,
randomFactor = 0.20
).withMaxNrOfRetries(3)
BackoffSupervisor.props(options)
} def create(port: Int): Unit = {
val config = ConfigFactory.parseString(s"akka.remote.netty.tcp.port=$port")
.withFallback(ConfigFactory.load()) val system = ActorSystem("cloud-pos-server", config)
system.actorOf(readerProps(true),"reader")
} } class ActionReader(trace: Boolean) extends Actor with LogSupport {
val cluster = Cluster(context.system)
val host = Cluster(context.system).selfAddress.host.get
implicit val nodeAddress: NodeAddress = NodeAddress(cluster.selfAddress.toString)
val readerId = "ActionReader"
log.stepOn = trace log.step(s"${nodeAddress.address}-${readerId}") override def preRestart(reason: Throwable, message: Option[Any]): Unit = {
super.preRestart(reason, message)
log.step(s"${nodeAddress.address}-${readerId} Restarting for $message ...")
} override def postRestart(reason: Throwable): Unit = {
super.postRestart(reason)
log.step(s"${nodeAddress.address}-${readerId} restarted for ${reason.getMessage}.")
} override def postStop(): Unit = {
log.step(s"${nodeAddress.address}-${readerId} stooped.")
} override def preStart(): Unit = {
log.step(s"${nodeAddress.address}-${readerId} Starting ...")
} var debugConfig: com.typesafe.config.Config = _
var debug: Boolean = _
try { debugConfig = ConfigFactory.load("pos.conf").getConfig("pos.server")
debug = debugConfig.getBoolean("debug")
}
catch {
case _ : Throwable => debug = false
} log.step(s"${nodeAddress.address}-${readerId} debug mode = $debug") implicit val debugMode = DebugMode(debug) override def receive: Receive = {
case PerformRead(pid, vchnum, bseq, eseq) =>
log.step(s"${nodeAddress.address}-${readerId} PerformRead($pid, $vchnum, $bseq, $eseq)")
readActions(host,bseq,eseq,pid,vchnum)(context.system,context.dispatcher,nodeAddress)
case msg @ _ =>
log.step(s"${nodeAddress.address}-${readerId} receive unsupported command:[$msg]")
} }
ReaderRouter.scala
package datatech.cloud.pos
import akka.actor._
import akka.routing._
import akka.cluster._
import com.typesafe.config.ConfigFactory class ReaderRouter extends Actor { val router = context.actorOf(FromConfig.props(), name = "readerRouter") def receive: Receive = {
case msg => router ! msg
} } object ReaderRouter {
var router: ActorRef = _ def props = Props(new ReaderRouter) def create(port: Int) = {
val config = ConfigFactory.parseString(s"akka.remote.netty.tcp.port=$port")
.withFallback(ConfigFactory.load()) val system = ActorSystem("cloud-pos-server",config) Cluster(system).registerOnMemberUp{
router = system.actorOf(props,"readerRouter")
} }
def getRouter = router
}
Reader.scala
package datatech.cloud.pos
import akka.actor._
import akka.stream.scaladsl._ import scala.util._
import akka._
import akka.persistence.query._
import akka.persistence.cassandra.query.scaladsl.CassandraReadJournal import scala.concurrent._
import akka.stream._
import sdp.logging._
import Actions._
import States._
import Messages._
import akka.cluster._
import DBWriter._ object Reader extends LogSupport { def readActions(cqlhost: String, startSeq: Long, endSeq: Long, persistenceId: String, vchnum: Int)(implicit sys: ActorSystem, ec: ExecutionContextExecutor, nodeAddress: NodeAddress) = {
implicit var vchState = VchStates().copy(num = vchnum)
implicit var vchItems = VchItems()
implicit var curTxnItem = TxnItem()
implicit val pid = PID(persistenceId)
implicit val mat = ActorMaterializer() val readerId = "ActionReader" // obtain read journal by plugin id
val readJournal =
PersistenceQuery(sys).readJournalFor[CassandraReadJournal](CassandraReadJournal.Identifier) // issue query to journal
val source: Source[EventEnvelope, NotUsed] =
readJournal.currentEventsByPersistenceId(persistenceId, startSeq, endSeq) // materialize stream, consuming events
val futureActions: Future[List[Any]] = source.runFold(List[Any]()) { (lstAny, evl) => evl.event :: lstAny } futureActions.onComplete {
case Success(txns) =>
log.step(s"${nodeAddress.address}-${readerId} recovered actions: $txns")
buildVoucher(txns)
case Failure(excpt) =>
log.error(s"${nodeAddress.address}-${readerId} read actions error: ${excpt.getMessage}") } def buildVoucher(actions: List[Any])= {
actions.reverse.foreach { txn =>
txn match {
case EndVoucher(_) =>
writeTxnsToDB(cqlhost,vchState.num,vchState.susp,vchItems.txnitems)(persistenceId,startSeq,endSeq)
mat.shutdown()
case ti@_ =>
curTxnItem = buildTxnItem(ti.asInstanceOf[Action])
val sts = updateState(ti.asInstanceOf[Action],0)
vchState = sts._1
vchItems = sts._2
}
}
}
} }
DBWriter.scala
package datatech.cloud.pos
import java.time.LocalDate
import java.time.format.DateTimeFormatter
import sdp.logging._
import Messages._
import com.datastax.driver.core._
import akka.actor.ActorSystem
import akka.stream.ActorMaterializer
import sdp.cql.engine._
import CQLEngine._
import CQLHelpers._
import com.typesafe.config._
import akka.stream.scaladsl._
import akka._
import scala.concurrent._ object DBWriter extends LogSupport {
var posConfig: com.typesafe.config.Config = _
def writeTxnsToDB(cqlhost: String, vchnum: Int, susp: Boolean, txns: List[TxnItem])(pid: String, bseq: Long, eseq: Long)(
implicit sys: ActorSystem, ec: ExecutionContextExecutor, mat: ActorMaterializer, nodeAddress: NodeAddress) = { val readerId = "DBWriter"
var cqlport: Int = 9042
try {
posConfig = ConfigFactory.load("pos.conf").getConfig("pos.cqlport")
cqlport = posConfig.getInt("cqlport")
}
catch {
case _ : Throwable => cqlport = 9042
} val cluster = new Cluster
.Builder()
.addContactPoints(cqlhost)
.withPort(cqlport)
.build() useJava8DateTime(cluster)
implicit val session = cluster.connect()
val insertVchLog = """
|insert into pos_on_cloud.vch_log(
|terminal,
|txndate,
|vchnum,
|begin_seq,
|end_seq)
|values(?,?,?,?,?)
|""".stripMargin val insertTxns = """
|insert into pos_on_cloud.txn_log(
|terminal,
|txndate,
|txntime,
|opr,
|num,
|seq,
|txntype,
|salestype,
|qty,
|price,
|amount,
|disc,
|dscamt,
|member,
|code,
|acct,
|dpt)
|values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
""".stripMargin val insertSusp = """
|insert into pos_on_cloud.txn_hold(
|terminal,
|txndate,
|txntime,
|opr,
|num,
|seq,
|txntype,
|salestype,
|qty,
|price,
|amount,
|disc,
|dscamt,
|member,
|code,
|acct,
|dpt)
|values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
""".stripMargin val vchParams: Seq[Object] = Seq(
pid.asInstanceOf[Object],
LocalDate.now.format(DateTimeFormatter.ofPattern("yyyyMMdd")).asInstanceOf[Object],
vchnum.asInstanceOf[Object],
bseq.asInstanceOf[Object],
eseq.asInstanceOf[Object]
) val vsToParams: Seq[Object] => Seq[Object] = vchParams => vchParams val actionStreamVs = CassandraActionStream(insertVchLog,vsToParams)
.setParallelism(2)
.setProcessOrder(false)
val actionFlowVs: Flow[Seq[Object],Seq[Object],NotUsed] = actionStreamVs.performOnRow val sinkVs = Sink.foreach[Seq[Object]]{ r =>
log.step(s"${nodeAddress.address}-${readerId} insert: $insertVchLog, values: ${r}")
}
// insert to vch_log
val stsVs = Source.fromIterator(() => Seq(vchParams).iterator).via(actionFlowVs).to(sinkVs).run() val insertTxn = if (susp) insertSusp else insertTxns val txnitemToParams: TxnItem => Seq[Object] = txn =>
(Seq(pid.asInstanceOf[Object]) ++ ccToList(txn)) val actionStreamTxn = CassandraActionStream(insertTxn,txnitemToParams)
.setParallelism(2)
.setProcessOrder(false)
val actionFlowTxn: Flow[TxnItem,TxnItem,NotUsed] = actionStreamTxn.performOnRow val sinkTxn = Sink.foreach[TxnItem]{ r =>
log.step(s"${nodeAddress.address}-${readerId} insert: $insertTxn, values: ${r}")
}
// insert to txn_???
val stsTxn = Source.fromIterator(() => txns.iterator).via(actionFlowTxn).to(sinkTxn).run()
} def ccToList(cc: Product) = {
val values = cc.productIterator
cc.getClass.getDeclaredFields.map(_ => (values.next).asInstanceOf[Object] ).toList
} }
ReaderDemo.scala
package datatech.cloud.pos
import akka.actor._ import datatech.cloud.pos.Messages.PerformRead
object ReaderDemo extends App {
ActionReader.create(2551) ActionReader.create(2552) ActionReader.create(2553) ReaderRouter.create(2558) scala.io.StdIn.readLine() val router = ReaderRouter.getRouter router ! PerformRead("1022",111,0,Long.MaxValue) scala.io.StdIn.readLine() router ! PerformRead("1022",222,0,Long.MaxValue) scala.io.StdIn.readLine() }
States.scala
package datatech.cloud.pos
import java.time.LocalDate
import java.time.LocalDateTime
import java.time.format.DateTimeFormatter import Messages._
import sdp.logging._ object Actions { implicit class FoldLeftWhile[A](trav: Seq[A]) {
def foldLeftWhile[B](z: B)(op: ((B,Boolean), A) => (B, Boolean)): B = {
def go(acc: (B, Boolean), l: Seq[A]): (B, Boolean) = l match {
case h +: t =>
val nacc = op(acc, h)
if (!nacc._2)
go(nacc, t)
else
nacc
case _ => acc
}
go((z, false), trav)._1
}
} case class ReadActions(startSeq: Int, endSeq: Int, persistenceId: String) sealed trait Action {}
case class LogOned(opr: String) extends Action
case object LogOffed extends Action
case class SuperOned(su: String) extends Action
case object SuperOffed extends Action
case class MemberOned(cardnum: String) extends Action
case object MemberOffed extends Action //remove member status for the voucher
case object RefundOned extends Action
case object RefundOffed extends Action
case object VoidOned extends Action
case object VoidOffed extends Action case class SalesLogged(acct: String, dpt: String, code: String, qty: Int, price: Int) extends Action
case class Subtotaled(level: Int) extends Action
case class Discounted(disctype: Int, grouped: Boolean, code: String, percent: Int) extends Action case class NewVoucher(vnum: Int) extends Action //新单, reminder for read-side to set new vnum
case class EndVoucher(vnum: Int) extends Action //单据终结标示
case object VoidVoucher extends Action case object SuspVoucher extends Action case class VoucherNumed(fnum: Int, tnum: Int) extends Action case class PaymentMade(acct: String, num: String, amount: Int) extends Action //settlement 结算支付 } object States extends LogSupport {
import Actions._ def setShowSteps(b: Boolean) = log.stepOn = b def buildTxnItem(evt: Action)(implicit vs: VchStates, vi: VchItems): TxnItem = evt match {
case LogOned(op) => TxnItem(vs).copy(
txntype = TXNTYPE.logon,
salestype = SALESTYPE.crd,
opr = op,
code = op
)
case LogOffed => TxnItem(vs).copy(
txntype = TXNTYPE.logon,
salestype = SALESTYPE.crd
)
case SuperOned(su) => TxnItem(vs).copy(
txntype = TXNTYPE.supon,
salestype = SALESTYPE.crd,
code = su
)
case SuperOffed => TxnItem(vs).copy(
txntype = TXNTYPE.supon,
salestype = SALESTYPE.crd
)
case MemberOned(cardnum) => TxnItem(vs).copy(
txntype = TXNTYPE.sales,
salestype = SALESTYPE.crd,
member = cardnum
)
case MemberOffed => TxnItem(vs).copy(
txntype = TXNTYPE.sales,
salestype = SALESTYPE.crd
)
case RefundOned => TxnItem(vs).copy(
txntype = TXNTYPE.refund
)
case RefundOffed => TxnItem(vs).copy(
txntype = TXNTYPE.refund
)
case VoidOned => TxnItem(vs).copy(
txntype = TXNTYPE.void
)
case VoidOffed => TxnItem(vs).copy(
txntype = TXNTYPE.void
)
case VoidVoucher => TxnItem(vs).copy(
txntype = TXNTYPE.voidall,
code = vs.num.toString,
acct = vs.num.toString
)
case SuspVoucher => TxnItem(vs).copy(
txntype = TXNTYPE.suspend,
code = vs.num.toString,
acct = vs.num.toString
)
case Subtotaled(level) =>
TxnItem(vs).copy(
txntype = TXNTYPE.sales,
salestype = SALESTYPE.sub
)
case Discounted(dt,gp,code,pct) => TxnItem(vs).copy(
txntype = TXNTYPE.sales,
salestype = SALESTYPE.dsc,
acct = code,
disc = pct
)
case PaymentMade(act,num,amt) => TxnItem(vs).copy(
txntype = TXNTYPE.sales,
salestype = SALESTYPE.ttl,
acct = act,
code = num,
amount = amt
) case SalesLogged(sacct,sdpt,scode,sqty,sprice) => TxnItem(vs).copy(
txntype = TXNTYPE.sales,
salestype = SALESTYPE.itm,
acct = sacct,
dpt = sdpt,
code = scode,
qty = sqty,
price = sprice,
amount = sprice * sqty,
dscamt = 0
)
case _ => TxnItem(vs)
} case class VchItems(txnitems: List[TxnItem] = Nil) { def noSales: Boolean = (txnitems.find(txn => txn.salestype == SALESTYPE.itm)).isEmpty def subTotal: (Int, Int, Int, Int) = txnitems.foldRight((0, 0, 0, 0)) { case (txn, b) =>
if (txn.salestype == SALESTYPE.itm && txn.txntype == TXNTYPE.sales)
b.copy(_1 = b._1 + 1, _2 = b._2 + txn.qty, _3 = b._3 + txn.amount, _4 = b._4 + txn.dscamt)
else b
} def groupTotal(level:Int): (Int, Int, Int, Int) = {
val gts = txnitems.foldLeftWhile((0, 0, 0, 0, 0)) { case (b,txn) =>
if (txn.salestype == SALESTYPE.itm && txn.txntype == TXNTYPE.sales)
((b._1._1 +1,b._1._2 + txn.qty, b._1._3 + txn.amount, b._1._4 + txn.dscamt, b._1._5),false)
else {
if (txn.salestype == SALESTYPE.sub) {
if (((b._1._5) + 1) >= level)
((b._1._1, b._1._2, b._1._3, b._1._4, b._1._5 + 1), true)
else
((b._1._1, b._1._2, b._1._3, b._1._4, b._1._5 + 1), false)
} else b
}
}
(gts._1,gts._2,gts._3,gts._4)
} def updateDisc(dt: Int, grouped: Boolean, disc: Int): (List[TxnItem],(Int,Int,Int,Int)) = {
//(salestype,（cnt,qty,amt,dsc),hassub,list）
val accu = txnitems.foldLeft((-1, (0,0,0,0), false, List[TxnItem]())) { case (b, txn) =>
var discAmt = 0
if ((b._1) < 0) {
if (txn.salestype == SALESTYPE.itm && txn.txntype == TXNTYPE.sales) {
if (txn.dscamt == 0)
((txn.salestype, (
(b._2._1) + 1,
(b._2._2) + txn.qty,
(b._2._3) + txn.amount,
(b._2._4) - (txn.amount * disc / 100)
), false, txn.copy(
dscamt = - (txn.amount * disc / 100)) :: (b._4)))
else {
dt match {
case DISCTYPE.duplicated =>
if (txn.dscamt != 0) {
((txn.salestype, (
(b._2._1) + 1,
(b._2._2) + txn.qty,
(b._2._3) + txn.amount,
(b._2._4) - (txn.amount + txn.dscamt) * disc / 100
), false, txn.copy(
dscamt = -(txn.amount + txn.dscamt) * disc / 100) :: (b._4)
))
} else {
((txn.salestype, (
(b._2._1) + 1,
(b._2._2) + txn.qty,
(b._2._3) + txn.amount,
(b._2._4) - txn.amount * disc / 100
), false, txn.copy(
dscamt = -txn.amount * disc / 100) :: (b._4)
))
}
case DISCTYPE.keep => ((txn.salestype, (
(b._2._1) + 1,
(b._2._2) + txn.qty,
(b._2._3) + txn.amount,
(b._2._4) + txn.dscamt), false, txn :: (b._4)))
case DISCTYPE.best =>
discAmt = -(txn.amount * disc / 100)
if (discAmt < txn.dscamt)
((txn.salestype, (
(b._2._1) + 1,
(b._2._2) + txn.qty,
(b._2._3) + txn.amount,
(b._2._4) + discAmt), false, txn.copy(
dscamt = discAmt
) :: (b._4)))
else
((txn.salestype, (
(b._2._1) + 1,
(b._2._2) + txn.qty,
(b._2._3) + txn.amount,
(b._2._4) + txn.dscamt), false, txn :: (b._4)))
}
} } else ((b._1,b._2,b._3,txn :: (b._4)))
} else {
if ((b._3))
(((b._1), (b._2), true, txn :: (b._4)))
else {
if (txn.salestype == SALESTYPE.sub) {
if (grouped)
(((b._1), (b._2), true, txn :: (b._4)))
else
(((b._1), (b._2), false, txn :: (b._4)))
} else {
if (txn.salestype == SALESTYPE.itm && txn.txntype == TXNTYPE.sales) {
dt match {
case DISCTYPE.duplicated =>
if (txn.dscamt != 0) {
((txn.salestype, (
(b._2._1) + 1,
(b._2._2) + txn.qty,
(b._2._3) + txn.amount,
(b._2._4) - (txn.amount + txn.dscamt) * disc / 100), false, txn.copy(
dscamt = -(txn.amount + txn.dscamt) * disc / 100) :: (b._4)
))
} else {
((txn.salestype, (
(b._2._1) + 1,
(b._2._2) + txn.qty,
(b._2._3) + txn.amount,
(b._2._4) - txn.amount * disc / 100), false, txn.copy(
dscamt = -(txn.amount * disc / 100)) :: (b._4)
))
}
case DISCTYPE.keep => ((txn.salestype, (
(b._2._1) + 1,
(b._2._2) + txn.qty,
(b._2._3) + txn.amount,
(b._2._4) + txn.dscamt), false, txn :: (b._4)))
case DISCTYPE.best =>
discAmt = -(txn.amount * disc / 100)
if (discAmt < txn.dscamt)
((txn.salestype, (
(b._2._1) + 1,
(b._2._2) + txn.qty,
(b._2._3) + txn.amount,
(b._2._4) + discAmt), false, txn.copy(
dscamt = discAmt
) :: (b._4)))
else
((txn.salestype, (
(b._2._1) + 1,
(b._2._2) + txn.qty,
(b._2._3) + txn.amount,
(b._2._4) + txn.dscamt), false, txn :: (b._4)))
}
}
else ((b._1, b._2, b._3, txn :: (b._4)))
}
}
} }
(accu._4.reverse,accu._2)
} def totalSales: Int = txnitems.foldRight(0) { case (txn, b) =>
if (txn.salestype == SALESTYPE.itm)
(txn.amount + txn.dscamt) + b
else b /*
val amt: Int = txn.salestype match {
case (SALESTYPE.plu | SALESTYPE.cat | SALESTYPE.brd | SALESTYPE.ra) => txn.amount + txn.dscamt
case _ => 0
}
amt + b */
} def totalPaid: Int = txnitems.foldRight(0) { case (txn, b) =>
if (txn.txntype == TXNTYPE.sales && txn.salestype == SALESTYPE.ttl)
txn.amount + b
else b
} def addItem(item: TxnItem): VchItems = VchItems((item :: txnitems)) //.reverse) } def LastSecOfDate(ldate: LocalDate): LocalDateTime = {
val dtStr = ldate.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")) + " 23:59:59"
LocalDateTime.parse(dtStr, DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"))
} def dateStr(dt: LocalDate): String = dt.format(DateTimeFormatter.ofPattern("yyyy-MM-dd")) def updateState(evt: Action, lastSeqNr: BigInt = 0)(implicit nodeAddress: NodeAddress, persistenceId: PID, state: VchStates, items: VchItems, curItem: TxnItem): (VchStates, VchItems) = {
val (vs, vi) = updateStateImpl(evt, lastSeqNr)
log.step(s"${nodeAddress.address}-${persistenceId.id} run updateState($evt, $lastSeqNr) with results state[$vs], txns[$vi].")
(vs, vi)
} def updateStateImpl(evt: Action, lastSeqNr: BigInt = 0)(implicit state: VchStates, items: VchItems, curItem: TxnItem): (VchStates, VchItems) = evt match {
case LogOned(csr) => (state.copy(seq = state.seq + 1, opr = csr, jseq = lastSeqNr), items)
case LogOffed => (state.copy(seq = state.seq + 1, opr = ""), items)
case RefundOned => (state.copy(seq = state.seq + 1, refd = true), items)
case RefundOffed => (state.copy(seq = state.seq + 1, refd = false), items)
case VoidOned => (state.copy(seq = state.seq + 1, void = true), items)
case VoidOffed => (state.copy(seq = state.seq + 1, void = false), items)
case SuperOned(suser) => (state.copy(seq = state.seq + 1, su = suser), items)
case SuperOffed => (state.copy(seq = state.seq + 1, su = ""), items)
case MemberOned(num) => (state.copy(seq = state.seq + 1, mbr = num), items)
case MemberOffed => (state.copy(seq = state.seq + 1, mbr = ""), items) case SalesLogged(_,_,_,_,_) => (state.copy(
seq = state.seq + 1)
, items.addItem(curItem)) case Subtotaled(level) =>
var subs = (0,0,0,0)
if (level == 0)
subs = items.subTotal
else
subs = items.groupTotal(level)
val (cnt, tqty, tamt, tdsc) = subs val subttlItem =
TxnItem(state).copy(
txntype = TXNTYPE.sales,
salestype = SALESTYPE.sub,
qty = tqty,
amount = tamt,
dscamt = tdsc,
price = cnt
)
(state.copy(
seq = state.seq + 1)
, items.addItem(subttlItem)) case Discounted(dt,gp,code,pct) =>
val (lstItems, accum) = items.updateDisc(dt,gp,pct)
val discItem = TxnItem(state).copy(
txntype = TXNTYPE.sales,
salestype = SALESTYPE.dsc,
acct = code,
disc = pct,
price = accum._1,
qty = accum._2,
amount = accum._3,
dscamt = accum._4
)
(state.copy(
seq = state.seq + 1)
, items.copy(txnitems = lstItems).addItem(discItem)) case PaymentMade(_,_,_) =>
val due = if (items.totalSales > 0) items.totalSales - items.totalPaid else items.totalSales + items.totalPaid
val bal = if (items.totalSales > 0) due - curItem.amount else due + curItem.amount
(state.copy(
seq = state.seq + 1,
due = (if ((curItem.amount.abs + items.totalPaid.abs) >= items.totalSales.abs) false else true)
)
,items.addItem(curItem.copy(
salestype = SALESTYPE.ttl,
price = due,
amount = curItem.amount,
dscamt = bal
))) case VoucherNumed(_, tnum) =>
val vi = items.copy(txnitems = items.txnitems.map { it => it.copy(num = tnum) })
(state.copy(seq = state.seq + 1, num = tnum), vi) case SuspVoucher => (state.copy(seq = state.seq + 1, susp = true), items) case VoidVoucher => (state.copy(seq = state.seq + 1, canc = true), items) case EndVoucher(vnum) => (state.nextVoucher.copy(jseq = lastSeqNr + 1), VchItems()) case _ => (state, items)
} }
Messages.scala
package datatech.cloud.pos import java.time.LocalDate
import java.time.LocalDateTime
import java.time.format.DateTimeFormatter
import java.util.Locale
import akka.cluster.sharding._ object Messages { val dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss", Locale.CHINA) sealed trait Command {} case class LogOn(opr: String, passwd: String) extends Command
case object LogOff extends Command
case class SuperOn(su: String, passwd: String) extends Command
case object SuperOff extends Command
case class MemberOn(cardnum: String, passwd: String) extends Command
case object MemberOff extends Command //remove member status for the voucher
case object RefundOn extends Command
case object RefundOff extends Command
case object VoidOn extends Command
case object VoidOff extends Command
case object VoidAll extends Command
case object Suspend extends Command case class VoucherNum(vnum: Int) extends Command case class LogSales(acct: String, dpt: String, code: String, qty: Int, price: Int) extends Command
case class Subtotal(level: Int) extends Command
case class Discount(disctype: Int, grouped: Boolean, code: String, percent: Int) extends Command case class Payment(acct: String, num: String, amount: Int) extends Command //settlement 结算支付 // read only command, no update event
case class Plu(itemCode: String) extends Command //read only
case object GetTxnItems extends Command /* discount type */
object DISCTYPE {
val duplicated: Int = 0
val best: Int = 1
val least: Int = 2
val keep: Int = 3
} /* result message returned to client on the wire */
object TXNTYPE {
val sales: Int = 0
val refund: Int = 1
val void: Int = 2
val voided: Int = 3
val voidall: Int = 4
val subtotal: Int = 5
val logon: Int = 6
val supon: Int = 7 // super user on/off
val suspend: Int = 8 } object SALESTYPE {
val itm: Int = 2
val sub: Int = 10
val ttl: Int = 11
val dsc: Int = 12
val crd: Int = 13
} case class TxnItem(
txndate: String = LocalDate.now.format(DateTimeFormatter.ofPattern("yyyyMMdd"))
,txntime: String = LocalDateTime.now.format(dateTimeFormatter).substring(11)
,opr: String = ""//工号
,num: Int = 0 //销售单号
,seq: Int = 1 //交易序号
,txntype: Int = TXNTYPE.sales//交易类型
,salestype: Int = SALESTYPE.itm //销售类型
,qty: Int = 1 //交易数量
,price: Int = 0 //单价（分）
,amount: Int = 0 //码洋（分）
,disc: Int = 0 //折扣率 (%)
,dscamt: Int = 0 //折扣额：负值 net实洋 = amount + dscamt
,member: String = "" //会员卡号
,code: String = "" //编号（商品、卡号...）
,acct: String = "" //账号
,dpt: String = "" //部类
)
object TxnItem {
def apply(vs: VchStates): TxnItem = TxnItem().copy(
opr = vs.opr,
num = vs.num,
seq = vs.seq,
member = vs.mbr
)
} case class VchStatus( //操作状态锁留给前端维护
qty: Int = 1,
refund: Boolean = false,
void: Boolean = false) case class VchStates(
opr: String = "", //收款员
jseq: BigInt = 0, //begin journal sequence for read-side replay
num: Int = 0, //当前单号
seq: Int = 0, //当前序号
void: Boolean = false, //取消模式
refd: Boolean = false, //退款模式
susp: Boolean = false, //挂单
canc: Boolean = false, //废单
due: Boolean = true, //当前余额
su: String = "",
mbr: String = ""
) { def nextVoucher : VchStates = VchStates().copy(
opr = this.opr,
jseq = this.jseq + 1,
num = this.num + 1
)
} object STATUS {
val OK: Int = 0
val FAIL: Int = -1
} case class POSResponse (sts: Int, msg: String, voucher: VchStates, txnItems: List[TxnItem]) /* message on the wire (exchanged message) */
val shardName = "POSShard" case class POSMessage(id: Long, cmd: Command) {
def shopId = id.toString.head.toString
def posId = id.toString
} val getPOSId: ShardRegion.ExtractEntityId = {
case posCommand: POSMessage => (posCommand.posId,posCommand.cmd)
}
val getShopId: ShardRegion.ExtractShardId = {
case posCommand: POSMessage => posCommand.shopId
} case object PassivatePOS //passivate message
case object FilteredOut
case class DebugMode(debug: Boolean)
case class NodeAddress(address: String)
case class PID(id: String)
case class PerformRead(pid: String, vchnum: Int, bseq: Long, eseq: Long)
}
cql/CassandraEngine.scala
package sdp.cql.engine import akka.NotUsed
import akka.stream.alpakka.cassandra.scaladsl._
import akka.stream.scaladsl._
import com.datastax.driver.core._
import com.google.common.util.concurrent.{FutureCallback, Futures, ListenableFuture}
import protobuf.bytes.Converter._
import sdp.logging.LogSupport
import sdp.result.DBOResult._ import scala.collection.JavaConverters._
import scala.collection.generic.CanBuildFrom
import scala.concurrent._ object CQLContext {
// Consistency Levels
type CONSISTENCY_LEVEL = Int
val ANY: CONSISTENCY_LEVEL = 0x0000
val ONE: CONSISTENCY_LEVEL = 0x0001
val TWO: CONSISTENCY_LEVEL = 0x0002
val THREE: CONSISTENCY_LEVEL = 0x0003
val QUORUM : CONSISTENCY_LEVEL = 0x0004
val ALL: CONSISTENCY_LEVEL = 0x0005
val LOCAL_QUORUM: CONSISTENCY_LEVEL = 0x0006
val EACH_QUORUM: CONSISTENCY_LEVEL = 0x0007
val LOCAL_ONE: CONSISTENCY_LEVEL = 0x000A
val LOCAL_SERIAL: CONSISTENCY_LEVEL = 0x000B
val SERIAL: CONSISTENCY_LEVEL = 0x000C def apply(): CQLUpdateContext = CQLUpdateContext(statements = Nil) def consistencyLevel: CONSISTENCY_LEVEL => ConsistencyLevel = consistency => {
consistency match {
case ALL => ConsistencyLevel.ALL
case ONE => ConsistencyLevel.ONE
case TWO => ConsistencyLevel.TWO
case THREE => ConsistencyLevel.THREE
case ANY => ConsistencyLevel.ANY
case EACH_QUORUM => ConsistencyLevel.EACH_QUORUM
case LOCAL_ONE => ConsistencyLevel.LOCAL_ONE
case QUORUM => ConsistencyLevel.QUORUM
case SERIAL => ConsistencyLevel.SERIAL
case LOCAL_SERIAL => ConsistencyLevel.LOCAL_SERIAL }
} }
case class CQLQueryContext(
statement: String,
parameters: Seq[Object] = Nil,
consistency: Option[CQLContext.CONSISTENCY_LEVEL] = None,
fetchSize: Int = 100
) { ctx =>
def setConsistencyLevel(_consistency: CQLContext.CONSISTENCY_LEVEL): CQLQueryContext =
ctx.copy(consistency = Some(_consistency))
def setFetchSize(pageSize: Int): CQLQueryContext =
ctx.copy(fetchSize = pageSize)
def setParameters(param: Seq[Object]): CQLQueryContext =
ctx.copy(parameters = param) def toProto = new sdp.grpc.services.ProtoCQLQuery(
statement = this.statement,
parameters = { if (this.parameters == Nil) None
else Some(sdp.grpc.services.ProtoAny(marshal(this.parameters))) },
consistency = this.consistency,
fetchSize = this.fetchSize
)
}
object CQLQueryContext {
def apply[M](stmt: String, param: Seq[Object]): CQLQueryContext = new CQLQueryContext(statement = stmt, parameters = param)
def fromProto(proto: sdp.grpc.services.ProtoCQLQuery) =
new CQLQueryContext(
statement = proto.statement,
parameters =
proto.parameters match {
case None => Nil
case Some(so) =>
if (so.value == _root_.com.google.protobuf.ByteString.EMPTY)
Nil
else
unmarshal[Seq[Object]](so.value)
},
consistency = proto.consistency,
fetchSize = proto.fetchSize
)
} case class CQLUpdateContext(
statements: Seq[String],
parameters: Seq[Seq[Object]] = Nil,
psize: Int = 0,
consistency: Option[CQLContext.CONSISTENCY_LEVEL] = None,
batch: Boolean = false
) extends LogSupport { ctx =>
def setBatch(bat: Boolean) = ctx.copy(batch = bat)
def setConsistencyLevel(_consistency: CQLContext.CONSISTENCY_LEVEL): CQLUpdateContext =
ctx.copy(consistency = Some(_consistency))
def setCommand(_statement: String, _psize: Int, _parameters: Object*): CQLUpdateContext = {
log.info(s"setCommand> setting: statement: ${_statement}, parameters: ${_parameters}")
var _params = Seq[Seq[Object]]()
if ( _psize > 0) {
if (_psize == 1)
_params = Seq(_parameters.asInstanceOf[Seq[Object]])
else
_params = _parameters.asInstanceOf[Seq[Seq[Object]]]
}
val nc = ctx.copy(statements = Seq(_statement), psize = _psize, parameters = _params)
log.info(s"setCommand> set: statements: ${nc.statements}, parameters: ${nc.parameters}")
nc
}
def appendCommand(_statement: String, _parameters: Object*): CQLUpdateContext = {
log.info(s"appendCommand> appending: statement: ${_statement}, parameters: ${_parameters}")
val nc = ctx.copy(statements = ctx.statements :+ _statement,
parameters = ctx.parameters ++ Seq(_parameters))
log.info(s"appendCommand> appended: statements: ${nc.statements}, parameters: ${nc.parameters}")
nc
} def toProto = new sdp.grpc.services.ProtoCQLUpdate(
statements = this.statements,
parameters = { if (this.parameters == Nil) None
else Some(sdp.grpc.services.ProtoAny(marshal(this.parameters))) },
consistency = this.consistency,
batch = Some(this.batch)
)
} object CQLUpdateContext {
def fromProto(proto: sdp.grpc.services.ProtoCQLUpdate) =
new CQLUpdateContext(
statements = proto.statements,
parameters =
proto.parameters match {
case None => Nil
case Some(so) =>
if (so.value == _root_.com.google.protobuf.ByteString.EMPTY)
Nil
else
unmarshal[Seq[Seq[Object]]](so.value)
},
consistency = proto.consistency,
batch = if(proto.batch == None) false else proto.batch.get
)
} object CQLEngine extends LogSupport {
import CQLContext._
import CQLHelpers._ import scala.concurrent.Await
import scala.concurrent.duration._ def fetchResult[C[_] <: TraversableOnce[_],A](ctx: CQLQueryContext, pageSize: Int = 100
,extractor: Row => A)(
implicit session: Session, cbf: CanBuildFrom[Nothing, A, C[A]]): DBOResult[C[A]]= { val prepStmt = session.prepare(ctx.statement) var boundStmt = prepStmt.bind()
var params: Seq[Object] = Nil
if (ctx.parameters != Nil) {
try {
params = processParameters(ctx.parameters)
boundStmt = prepStmt.bind(params: _*)
}
catch {
case e: Exception =>
log.error(s"fetchResult> prepStmt.bind error: ${e.getMessage}")
Left(new RuntimeException(s"fetchResult> prepStmt.bind Error: ${e.getMessage}"))
}
}
log.info(s"fetchResult> statement: ${prepStmt.getQueryString}, parameters: ${params}") try {
ctx.consistency.foreach { consistency =>
boundStmt.setConsistencyLevel(consistencyLevel(consistency))
} val resultSet = session.execute(boundStmt.setFetchSize(pageSize))
val rows = resultSet.asScala.view.map(extractor).to[C]
valueToDBOResult(rows)
/*
val ores = if(rows.isEmpty) None else Some(rows)
optionToDBOResult(ores: Option[C[A]]) */
}
catch {
case e: Exception =>
log.error(s"fetchResult> runtime error: ${e.getMessage}")
Left(new RuntimeException(s"fetchResult> Error: ${e.getMessage}"))
}
} def fetchResultPage[C[_] <: TraversableOnce[_],A](ctx: CQLQueryContext, pageSize: Int = 100
,extractor: Row => A)(
implicit session: Session, cbf: CanBuildFrom[Nothing, A, C[A]]): (ResultSet, C[A])= { val prepStmt = session.prepare(ctx.statement) var boundStmt = prepStmt.bind()
var params: Seq[Object] = Nil
if (ctx.parameters != Nil) {
params = processParameters(ctx.parameters)
boundStmt = prepStmt.bind(params:_*)
}
log.info(s"fetchResultPage> statement: ${prepStmt.getQueryString}, parameters: ${params}") ctx.consistency.foreach {consistency =>
boundStmt.setConsistencyLevel(consistencyLevel(consistency))} val resultSet = session.execute(boundStmt.setFetchSize(pageSize))
(resultSet,(resultSet.asScala.view.map(extractor)).to[C])
} def fetchMorePages[C[_] <: TraversableOnce[_],A](resultSet: ResultSet, timeOut: Duration)(
extractor: Row => A)(implicit ec: ExecutionContext, cbf: CanBuildFrom[Nothing, A, C[A]]): (ResultSet,Option[C[A]]) =
if (resultSet.isFullyFetched) {
(resultSet, None)
} else {
try {
val result = Await.result((resultSet.fetchMoreResults()).asScala, timeOut)
(result, Some((result.asScala.view.map(extractor)).to[C]))
} catch { case e: Throwable => (resultSet, None) }
} def cqlExecute(ctx: CQLUpdateContext)(
implicit session: Session, ec: ExecutionContext): DBOResult[Boolean] = {
var ctxparameters = Seq[Seq[Object]]()
if (ctx.parameters != Nil)
if (ctx.parameters.head != Nil) {
ctxparameters = ctx.parameters.asInstanceOf[Seq[Seq[Seq[Object]]]].head
} var invalidBat = false
if ( ctx.batch ) {
if (ctxparameters == Nil)
invalidBat = true
else if (ctxparameters.size < 2)
invalidBat = true;
}
if (!ctx.batch || invalidBat) {
if(invalidBat)
log.warn(s"cqlExecute> batch update must have at least 2 sets of parameters! change to single-command.") if (ctx.statements.size == 1 && ctx.psize <= 1) {
var param: Seq[Seq[Object]] = Nil
if (ctxparameters != Nil)
param = ctxparameters
log.info(s"cqlExecute> single-command: statement: ${ctx.statements.head} parameters: ${param}")
cqlSingleUpdate(ctx.consistency, ctx.statements.head, param)
}
else {
var params: Seq[Seq[Object]] = ctxparameters
var ctxstatements = ctx.statements
if (ctxparameters.size < ctx.statements.size) {
log.warn(s"cqlExecute> fewer parameters than statements! pad with 'Nil'.")
val pnils = Seq.fill(ctx.statements.length - ctxparameters.size)(Nil)
params = ctxparameters ++ pnils
}
else {
if (ctx.statements.size < ctxparameters.size) {
log.warn(s"cqlExecute> fewer statements than parameters! pad with 'head'.")
val heads = Seq.fill(ctxparameters.size - ctx.statements.size)(ctx.statements.head)
ctxstatements = ctx.statements ++ heads
}
} val commands: Seq[(String, Seq[Object])] = ctxstatements zip params
log.info(s"cqlExecute> multi-commands: ${commands}")
/*
//using sequence to flip List[Future[Boolean]] => Future[List[Boolean]]
//therefore, make sure no command replies on prev command effect
val lstCmds: List[Future[Boolean]] = commands.map { case (stmt,param) =>
cqlSingleUpdate(ctx.consistency, stmt, param)
}.toList val futList = Future.sequence(lstCmds).map(_ => true) //must map to execute
*/
/*
//using traverse to have some degree of parallelism = max(runtimes)
//therefore, make sure no command replies on prev command effect
val futList = Future.traverse(commands) { case (stmt,param) =>
cqlSingleUpdate(ctx.consistency, stmt, param)
}.map(_ => true) Await.result(futList, 3 seconds)
Future.successful(true)
*/
// run sync directly
try {
commands.foreach { case (stm, pars) =>
cqlExecuteSync(ctx.consistency, stm, pars)
}
Right(true)
}
catch {
case e: Exception =>
log.error(s"cqlExecute> runtime error: ${e.getMessage}")
Left(new RuntimeException(s"cqlExecute> Error: ${e.getMessage}"))
}
}
}
else
cqlBatchUpdate(ctx)
}
def cqlSingleUpdate(cons: Option[CQLContext.CONSISTENCY_LEVEL],stmt: String, params: Seq[Seq[Object]])(
implicit session: Session, ec: ExecutionContext): DBOResult[Boolean] = { val prepStmt = session.prepare(stmt) var boundStmt = prepStmt.bind()
var pars: Seq[Seq[Object]] = Nil
if (params != Nil) {
try {
pars = params.map(processParameters(_))
boundStmt = prepStmt.bind(pars.head: _*)
}
catch {
case e: Exception =>
log.error(s"cqlSingleUpdate> prepStmt.bind error: ${e.getMessage}")
Left(new RuntimeException(s"cqlSingleUpdate> prepStmt.bind Error: ${e.getMessage}"))
}
}
log.info(s"cqlSingleUpdate> statement: ${prepStmt.getQueryString}, parameters: ${pars}") try {
cons.foreach { consistency =>
boundStmt.setConsistencyLevel(consistencyLevel(consistency))
}
val res = session.execute(boundStmt) //executeAsync(boundStmt).map(_.wasApplied())
Right(res.wasApplied())
}
catch {
case e: Exception =>
log.error(s"cqlExecute> runtime error: ${e.getMessage}")
Left(new RuntimeException(s"cqlExecute> Error: ${e.getMessage}"))
}
} def cqlExecuteSync(cons: Option[CQLContext.CONSISTENCY_LEVEL],stmt: String, params: Seq[Object])(
implicit session: Session, ec: ExecutionContext): Boolean = { val prepStmt = session.prepare(stmt) var boundStmt = prepStmt.bind()
var pars: Seq[Object] = Nil
if (params != Nil) {
pars = processParameters(params)
boundStmt = prepStmt.bind(pars: _*)
}
log.info(s"cqlExecuteSync> statement: ${prepStmt.getQueryString}, parameters: ${pars}") cons.foreach { consistency =>
boundStmt.setConsistencyLevel(consistencyLevel(consistency))
}
session.execute(boundStmt).wasApplied() } def cqlBatchUpdate(ctx: CQLUpdateContext)(
implicit session: Session, ec: ExecutionContext): DBOResult[Boolean] = {
var ctxparameters = Seq[Seq[Object]]()
if(ctx.parameters != Nil)
if (ctx.parameters.head != Nil)
ctxparameters = ctx.parameters.asInstanceOf[Seq[Seq[Seq[Object]]]].head
var params: Seq[Seq[Object]] = ctxparameters
var ctxstatements = ctx.statements
if (ctxparameters.size < ctx.statements.size) {
log.warn(s"cqlBatchUpdate> fewer parameters than statements! pad with 'Nil'.")
val pnils = Seq.fill(ctx.statements.length - ctxparameters.size)(Nil)
params = ctxparameters ++ pnils
}
else {
if (ctx.statements.size < ctxparameters.size) {
log.warn(s"cqlBatchUpdate> fewer statements than parameters! pad with 'head'.")
val heads = Seq.fill(ctxparameters.size - ctx.statements.size)(ctx.statements.head)
ctxstatements = ctx.statements ++ heads
}
} val commands: Seq[(String, Seq[Object])] = ctxstatements zip params
log.info(s"cqlBatchUpdate> batch-commands: ${commands}") var batch = new BatchStatement()
try {
commands.foreach { cmd =>
val prepStmt = session.prepare(cmd._1)
log.info(s"cqlBatchUpdate> batch with statement: ${cmd._1}, raw parameter: ${cmd._2}")
if (cmd._2 == Nil) {
val pars = processParameters(cmd._2)
log.info(s"cqlBatchUpdate> batch with cooked parameters: ${pars}")
batch.add(prepStmt.bind(pars: _*))
} else {
log.info(s"cqlBatchUpdate> batch with no parameter")
batch.add(prepStmt.bind())
}
}
ctx.consistency.foreach { consistency =>
batch.setConsistencyLevel(consistencyLevel(consistency))
}
val res = session.execute(batch) //session.executeAsync(batch).map(_.wasApplied())
Right(res.wasApplied())
}
catch {
case e: Exception =>
log.error(s"cqlBatchUpdate> runtime error: ${e.getMessage}")
Left(new RuntimeException(s"cqlBatchUpdate> Error: ${e.getMessage}"))
} } def cassandraStream[A](ctx: CQLQueryContext,extractor: Row => A)
(implicit session: Session, ec: ExecutionContextExecutor): Source[A,NotUsed] = { val prepStmt = session.prepare(ctx.statement)
var boundStmt = prepStmt.bind()
val params = processParameters(ctx.parameters)
boundStmt = prepStmt.bind(params:_*)
ctx.consistency.foreach {consistency =>
boundStmt.setConsistencyLevel(consistencyLevel(consistency))} log.info(s"cassandraStream> statement: ${prepStmt.getQueryString}, parameters: ${params}")
CassandraSource(boundStmt.setFetchSize(ctx.fetchSize)).map(extractor)
} case class CassandraActionStream[R](parallelism: Int = 1, processInOrder: Boolean = true,
statement: String, prepareParams: R => Seq[Object],
consistency: Option[CQLContext.CONSISTENCY_LEVEL] = None){ cas =>
def setParallelism(parLevel: Int): CassandraActionStream[R] = cas.copy(parallelism=parLevel)
def setProcessOrder(ordered: Boolean): CassandraActionStream[R] = cas.copy(processInOrder = ordered)
def setConsistencyLevel(_consistency: CQLContext.CONSISTENCY_LEVEL): CassandraActionStream[R] =
cas.copy(consistency = Some(_consistency)) def perform(r: R)(implicit session: Session, ec: ExecutionContext) = {
var prepStmt = session.prepare(statement)
var boundStmt = prepStmt.bind()
val params = processParameters(prepareParams(r))
boundStmt = prepStmt.bind(params: _*)
consistency.foreach { cons =>
boundStmt.setConsistencyLevel(CQLContext.consistencyLevel(cons))
}
log.info(s"CassandraActionStream.perform> statement: ${prepStmt.getQueryString}, parameters: ${params}")
import monix.eval.Task
import monix.execution.Scheduler.Implicits.global
session.execute(boundStmt)
Task.now {r}.runToFuture
//session.executeAsync(boundStmt).map(_ => r)
} def performOnRow(implicit session: Session, ec: ExecutionContext): Flow[R, R, NotUsed] =
if (processInOrder)
Flow[R].mapAsync(parallelism)(perform)
else
Flow[R].mapAsyncUnordered(parallelism)(perform) def unloggedBatch[K](statementBinder: (
R, PreparedStatement) => BoundStatement,partitionKey: R => K)(
implicit session: Session, ec: ExecutionContext): Flow[R, R, NotUsed] = {
val preparedStatement = session.prepare(statement)
log.info(s"CassandraActionStream.unloggedBatch> statement: ${preparedStatement.getQueryString}")
CassandraFlow.createUnloggedBatchWithPassThrough[R, K](
parallelism,
preparedStatement,
statementBinder,
partitionKey)
} }
object CassandraActionStream {
def apply[R](_statement: String, params: R => Seq[Object]): CassandraActionStream[R] =
new CassandraActionStream[R]( statement=_statement, prepareParams = params)
} } object CQLHelpers extends LogSupport {
import java.io._
import java.nio.ByteBuffer
import java.nio.file._
import java.time.Instant
import scala.util.Try import akka.stream._
import akka.stream.scaladsl._
import com.datastax.driver.core.LocalDate
import com.datastax.driver.extras.codecs.jdk8.InstantCodec
import java.util.concurrent.Executor
/*
implicit def listenableFutureToFuture[T](
listenableFuture: ListenableFuture[T]): Future[T] = {
val promise = Promise[T]()
Futures.addCallback(listenableFuture, new FutureCallback[T] {
def onFailure(error: Throwable): Unit = {
promise.failure(error)
()
}
def onSuccess(result: T): Unit = {
promise.success(result)
()
}
})
promise.future
} */ implicit def listenableFutureToFuture[A](lf: ListenableFuture[A])(implicit executionContext: ExecutionContext): Future[A] = {
val promise = Promise[A]
lf.addListener(new Runnable {
def run() = promise.complete(Try(lf.get()))
}, executionContext.asInstanceOf[Executor])
promise.future
} implicit class ListenableFutureConverter[A](val lf: ListenableFuture[A]) extends AnyVal {
def asScala(implicit ec: ExecutionContext): Future[A] = {
val promise = Promise[A]
lf.addListener(new Runnable {
def run() = promise.complete(Try(lf.get()))
}, ec.asInstanceOf[Executor])
promise.future
}
}
/*
implicit def toScalaFuture[A](a: ListenableFuture[A])(implicit ec: ExecutionContext): Future[A] = {
val promise = Promise[A]()
a.addListener(new Runnable {
def run() = {
try {
promise.success(a.get)
} catch {
case ex: ExecutionException => promise.failure(ex.getCause)
case ex => promise.failure(ex)
}
}
}, ec.asInstanceOf[Executor])
promise.future
} */ case class CQLDate(year: Int, month: Int, day: Int)
case object CQLTodayDate
case class CQLDateTime(year: Int, Month: Int,
day: Int, hour: Int, minute: Int, second: Int, millisec: Int = 0)
case object CQLDateTimeNow def cqlGetDate(dateToConvert: java.util.Date): java.time.LocalDate =
dateToConvert.toInstant()
.atZone(java.time.ZoneId.systemDefault())
.toLocalDate() def cqlGetTime(dateToConvert: java.util.Date): java.time.LocalTime =
dateToConvert.toInstant()
.atZone(java.time.ZoneId.systemDefault())
.toLocalTime() def cqlGetTimestamp(dateToConvert: java.util.Date): java.time.LocalDateTime=
new java.sql.Timestamp(
dateToConvert.getTime()
).toLocalDateTime() def processParameters(params: Seq[Object]): Seq[Object] = {
import java.time.{Clock, ZoneId}
log.info(s"[processParameters] input: ${params}")
val outParams = params.map { obj =>
obj match {
case CQLDate(yy, mm, dd) => LocalDate.fromYearMonthDay(yy, mm, dd)
case CQLTodayDate =>
val today = java.time.LocalDate.now()
LocalDate.fromYearMonthDay(today.getYear, today.getMonth.getValue, today.getDayOfMonth)
case CQLDateTimeNow => Instant.now(Clock.system(ZoneId.of("EST", ZoneId.SHORT_IDS)))
case CQLDateTime(yy, mm, dd, hr, ms, sc, mi) =>
Instant.parse(f"$yy%4d-$mm%2d-$dd%2dT$hr%2d:$ms%2d:$sc%2d$mi%3d")
case p@_ => p
}
}
log.info(s"[processParameters] output: ${params}")
outParams
}
class ByteBufferInputStream(buf: ByteBuffer) extends InputStream {
override def read: Int = {
if (!buf.hasRemaining) return -1
buf.get
} override def read(bytes: Array[Byte], off: Int, len: Int): Int = {
val length: Int = Math.min(len, buf.remaining)
buf.get(bytes, off, length)
length
}
}
object ByteBufferInputStream {
def apply(buf: ByteBuffer): ByteBufferInputStream = {
new ByteBufferInputStream(buf)
}
}
class FixsizedByteBufferOutputStream(buf: ByteBuffer) extends OutputStream { override def write(b: Int): Unit = {
buf.put(b.toByte)
} override def write(bytes: Array[Byte], off: Int, len: Int): Unit = {
buf.put(bytes, off, len)
}
}
object FixsizedByteBufferOutputStream {
def apply(buf: ByteBuffer) = new FixsizedByteBufferOutputStream(buf)
}
class ExpandingByteBufferOutputStream(var buf: ByteBuffer, onHeap: Boolean) extends OutputStream { private val increasing = ExpandingByteBufferOutputStream.DEFAULT_INCREASING_FACTOR override def write(b: Array[Byte], off: Int, len: Int): Unit = {
val position = buf.position
val limit = buf.limit
val newTotal: Long = position + len
if(newTotal > limit){
var capacity = (buf.capacity * increasing)
while(capacity <= newTotal){
capacity = (capacity*increasing)
}
increase(capacity.toInt)
} buf.put(b, 0, len)
} override def write(b: Int): Unit= {
if (!buf.hasRemaining) increase((buf.capacity * increasing).toInt)
buf.put(b.toByte)
}
protected def increase(newCapacity: Int): Unit = {
buf.limit(buf.position)
buf.rewind
val newBuffer =
if (onHeap) ByteBuffer.allocate(newCapacity)
else ByteBuffer.allocateDirect(newCapacity)
newBuffer.put(buf)
buf.clear
buf = newBuffer
}
def size: Long = buf.position
def capacity: Long = buf.capacity
def byteBuffer: ByteBuffer = buf
}
object ExpandingByteBufferOutputStream {
val DEFAULT_INCREASING_FACTOR = 1.5f
def apply(size: Int, increasingBy: Float, onHeap: Boolean) = {
if (increasingBy <= 1) throw new IllegalArgumentException("Increasing Factor must be greater than 1.0")
val buffer: ByteBuffer =
if (onHeap) ByteBuffer.allocate(size)
else ByteBuffer.allocateDirect(size)
new ExpandingByteBufferOutputStream(buffer,onHeap)
}
def apply(size: Int): ExpandingByteBufferOutputStream = {
apply(size, ExpandingByteBufferOutputStream.DEFAULT_INCREASING_FACTOR, false)
} def apply(size: Int, onHeap: Boolean): ExpandingByteBufferOutputStream = {
apply(size, ExpandingByteBufferOutputStream.DEFAULT_INCREASING_FACTOR, onHeap)
} def apply(size: Int, increasingBy: Float): ExpandingByteBufferOutputStream = {
apply(size, increasingBy, false)
} }
def cqlFileToBytes(fileName: String): ByteBuffer = {
val fis = new FileInputStream(fileName)
val b = new Array[Byte](fis.available + 1)
val length = b.length
fis.read(b)
ByteBuffer.wrap(b)
}
def cqlBytesToFile(bytes: ByteBuffer, fileName: String)(
implicit mat: Materializer): Future[IOResult] = {
val source = StreamConverters.fromInputStream(() => ByteBufferInputStream(bytes))
source.runWith(FileIO.toPath(Paths.get(fileName)))
}
def cqlDateTimeString(date: java.util.Date, fmt: String): String = {
val outputFormat = new java.text.SimpleDateFormat(fmt)
outputFormat.format(date)
}
def useJava8DateTime(cluster: Cluster) = {
//for jdk8 datetime format
cluster.getConfiguration().getCodecRegistry()
.register(InstantCodec.instance)
}
}
mgo/MGOProtoConversion.scala
package sdp.mongo.engine
import org.mongodb.scala.bson.collection.immutable.Document
import org.bson.conversions.Bson
import sdp.grpc.services._
import protobuf.bytes.Converter._
import MGOClasses._
import MGOAdmins._
import MGOCommands._
import org.bson.BsonDocument
import org.bson.codecs.configuration.CodecRegistry
import org.mongodb.scala.bson.codecs.DEFAULT_CODEC_REGISTRY
import org.mongodb.scala.FindObservable object MGOProtoConversion { type MGO_COMMAND_TYPE = Int
val MGO_COMMAND_FIND = 0
val MGO_COMMAND_COUNT = 20
val MGO_COMMAND_DISTICT = 21
val MGO_COMMAND_DOCUMENTSTREAM = 1
val MGO_COMMAND_AGGREGATE = 2
val MGO_COMMAND_INSERT = 3
val MGO_COMMAND_DELETE = 4
val MGO_COMMAND_REPLACE = 5
val MGO_COMMAND_UPDATE = 6 val MGO_ADMIN_DROPCOLLECTION = 8
val MGO_ADMIN_CREATECOLLECTION = 9
val MGO_ADMIN_LISTCOLLECTION = 10
val MGO_ADMIN_CREATEVIEW = 11
val MGO_ADMIN_CREATEINDEX = 12
val MGO_ADMIN_DROPINDEXBYNAME = 13
val MGO_ADMIN_DROPINDEXBYKEY = 14
val MGO_ADMIN_DROPALLINDEXES = 15 case class AdminContext(
tarName: String = "",
bsonParam: Seq[Bson] = Nil,
options: Option[Any] = None,
objName: String = ""
){
def toProto = sdp.grpc.services.ProtoMGOAdmin(
tarName = this.tarName,
bsonParam = this.bsonParam.map {b => sdp.grpc.services.ProtoMGOBson(marshal(b))},
objName = this.objName,
options = this.options.map(b => ProtoAny(marshal(b))) )
} object AdminContext {
def fromProto(msg: sdp.grpc.services.ProtoMGOAdmin) = new AdminContext(
tarName = msg.tarName,
bsonParam = msg.bsonParam.map(b => unmarshal[Bson](b.bson)),
objName = msg.objName,
options = msg.options.map(b => unmarshal[Any](b.value))
)
} case class Context(
dbName: String = "",
collName: String = "",
commandType: MGO_COMMAND_TYPE,
bsonParam: Seq[Bson] = Nil,
resultOptions: Seq[ResultOptions] = Nil,
options: Option[Any] = None,
documents: Seq[Document] = Nil,
targets: Seq[String] = Nil,
only: Boolean = false,
adminOptions: Option[AdminContext] = None
){ def toProto = new sdp.grpc.services.ProtoMGOContext(
dbName = this.dbName,
collName = this.collName,
commandType = this.commandType,
bsonParam = this.bsonParam.map(bsonToProto),
resultOptions = this.resultOptions.map(_.toProto),
options = { if(this.options == None)
None //Some(ProtoAny(com.google.protobuf.ByteString.EMPTY))
else
Some(ProtoAny(marshal(this.options.get))) },
documents = this.documents.map(d => sdp.grpc.services.ProtoMGODocument(marshal(d))),
targets = this.targets,
only = Some(this.only),
adminOptions = this.adminOptions.map(_.toProto)
) } object MGODocument {
def fromProto(msg: sdp.grpc.services.ProtoMGODocument): Document =
unmarshal[Document](msg.document)
def toProto(doc: Document): sdp.grpc.services.ProtoMGODocument =
new ProtoMGODocument(marshal(doc))
} object MGOProtoMsg {
def fromProto(msg: sdp.grpc.services.ProtoMGOContext) = new Context(
dbName = msg.dbName,
collName = msg.collName,
commandType = msg.commandType,
bsonParam = msg.bsonParam.map(protoToBson),
resultOptions = msg.resultOptions.map(r => ResultOptions.fromProto(r)),
options = msg.options.map(a => unmarshal[Any](a.value)),
documents = msg.documents.map(doc => unmarshal[Document](doc.document)),
targets = msg.targets,
adminOptions = msg.adminOptions.map(ado => AdminContext.fromProto(ado))
)
} def bsonToProto(bson: Bson) =
ProtoMGOBson(marshal(bson.toBsonDocument(
classOf[org.mongodb.scala.bson.collection.immutable.Document],DEFAULT_CODEC_REGISTRY))) def protoToBson(proto: ProtoMGOBson): Bson = new Bson {
val bsdoc = unmarshal[BsonDocument](proto.bson)
override def toBsonDocument[TDocument](documentClass: Class[TDocument], codecRegistry: CodecRegistry): BsonDocument = bsdoc
} def ctxFromProto(proto: ProtoMGOContext): MGOContext = proto.commandType match {
case MGO_COMMAND_FIND => {
var ctx = new MGOContext(
dbName = proto.dbName,
collName = proto.collName,
actionType = MGO_QUERY,
action = Some(Find())
)
def toResultOption(rts: Seq[ProtoMGOResultOption]): FindObservable[Document] => FindObservable[Document] = findObj =>
rts.foldRight(findObj)((a,b) => ResultOptions.fromProto(a).toFindObservable(b)) (proto.bsonParam, proto.resultOptions, proto.only) match {
case (Nil, Nil, None) => ctx
case (Nil, Nil, Some(b)) => ctx.setCommand(Find(firstOnly = b))
case (bp,Nil,None) => ctx.setCommand(
Find(filter = Some(protoToBson(bp.head))))
case (bp,Nil,Some(b)) => ctx.setCommand(
Find(filter = Some(protoToBson(bp.head)), firstOnly = b))
case (bp,fo,None) => {
ctx.setCommand(
Find(filter = Some(protoToBson(bp.head)),
andThen = fo.map(ResultOptions.fromProto)
))
}
case (bp,fo,Some(b)) => {
ctx.setCommand(
Find(filter = Some(protoToBson(bp.head)),
andThen = fo.map(ResultOptions.fromProto),
firstOnly = b))
}
case _ => ctx
}
}
case MGO_COMMAND_COUNT => {
var ctx = new MGOContext(
dbName = proto.dbName,
collName = proto.collName,
actionType = MGO_QUERY,
action = Some(Count())
)
(proto.bsonParam, proto.options) match {
case (Nil, None) => ctx
case (bp, None) => ctx.setCommand(
Count(filter = Some(protoToBson(bp.head)))
)
case (Nil,Some(o)) => ctx.setCommand(
Count(options = Some(unmarshal[Any](o.value)))
)
case _ => ctx
}
}
case MGO_COMMAND_DISTICT => {
var ctx = new MGOContext(
dbName = proto.dbName,
collName = proto.collName,
actionType = MGO_QUERY,
action = Some(Distict(fieldName = proto.targets.head))
)
(proto.bsonParam) match {
case Nil => ctx
case bp: Seq[ProtoMGOBson] => ctx.setCommand(
Distict(fieldName = proto.targets.head,filter = Some(protoToBson(bp.head)))
)
case _ => ctx
}
}
case MGO_COMMAND_AGGREGATE => {
new MGOContext(
dbName = proto.dbName,
collName = proto.collName,
actionType = MGO_QUERY,
action = Some(Aggregate(proto.bsonParam.map(p => protoToBson(p))))
)
}
case MGO_ADMIN_LISTCOLLECTION => {
new MGOContext(
dbName = proto.dbName,
collName = proto.collName,
actionType = MGO_QUERY,
action = Some(ListCollection(proto.dbName)))
}
case MGO_COMMAND_INSERT => {
var ctx = new MGOContext(
dbName = proto.dbName,
collName = proto.collName,
actionType = MGO_UPDATE,
action = Some(Insert(
newdocs = proto.documents.map(doc => unmarshal[Document](doc.document))))
)
proto.options match {
case None => ctx
case Some(o) => ctx.setCommand(Insert(
newdocs = proto.documents.map(doc => unmarshal[Document](doc.document)),
options = Some(unmarshal[Any](o.value)))
)
}
}
case MGO_COMMAND_DELETE => {
var ctx = new MGOContext(
dbName = proto.dbName,
collName = proto.collName,
actionType = MGO_UPDATE,
action = Some(Delete(
filter = protoToBson(proto.bsonParam.head)))
)
(proto.options, proto.only) match {
case (None,None) => ctx
case (None,Some(b)) => ctx.setCommand(Delete(
filter = protoToBson(proto.bsonParam.head),
onlyOne = b))
case (Some(o),None) => ctx.setCommand(Delete(
filter = protoToBson(proto.bsonParam.head),
options = Some(unmarshal[Any](o.value)))
)
case (Some(o),Some(b)) => ctx.setCommand(Delete(
filter = protoToBson(proto.bsonParam.head),
options = Some(unmarshal[Any](o.value)),
onlyOne = b)
)
}
}
case MGO_COMMAND_REPLACE => {
var ctx = new MGOContext(
dbName = proto.dbName,
collName = proto.collName,
actionType = MGO_UPDATE,
action = Some(Replace(
filter = protoToBson(proto.bsonParam.head),
replacement = unmarshal[Document](proto.documents.head.document)))
)
proto.options match {
case None => ctx
case Some(o) => ctx.setCommand(Replace(
filter = protoToBson(proto.bsonParam.head),
replacement = unmarshal[Document](proto.documents.head.document),
options = Some(unmarshal[Any](o.value)))
)
}
}
case MGO_COMMAND_UPDATE => {
var ctx = new MGOContext(
dbName = proto.dbName,
collName = proto.collName,
actionType = MGO_UPDATE,
action = Some(Update(
filter = protoToBson(proto.bsonParam.head),
update = protoToBson(proto.bsonParam.tail.head)))
)
(proto.options, proto.only) match {
case (None,None) => ctx
case (None,Some(b)) => ctx.setCommand(Update(
filter = protoToBson(proto.bsonParam.head),
update = protoToBson(proto.bsonParam.tail.head),
onlyOne = b))
case (Some(o),None) => ctx.setCommand(Update(
filter = protoToBson(proto.bsonParam.head),
update = protoToBson(proto.bsonParam.tail.head),
options = Some(unmarshal[Any](o.value)))
)
case (Some(o),Some(b)) => ctx.setCommand(Update(
filter = protoToBson(proto.bsonParam.head),
update = protoToBson(proto.bsonParam.tail.head),
options = Some(unmarshal[Any](o.value)),
onlyOne = b)
)
}
}
case MGO_ADMIN_DROPCOLLECTION =>
new MGOContext(
dbName = proto.dbName,
collName = proto.collName,
actionType = MGO_ADMIN,
action = Some(DropCollection(proto.collName))
)
case MGO_ADMIN_CREATECOLLECTION => {
var ctx = new MGOContext(
dbName = proto.dbName,
collName = proto.collName,
actionType = MGO_ADMIN,
action = Some(CreateCollection(proto.collName))
)
proto.options match {
case None => ctx
case Some(o) => ctx.setCommand(CreateCollection(proto.collName,
options = Some(unmarshal[Any](o.value)))
)
}
}
case MGO_ADMIN_CREATEVIEW => {
var ctx = new MGOContext(
dbName = proto.dbName,
collName = proto.collName,
actionType = MGO_ADMIN,
action = Some(CreateView(viewName = proto.targets.head,
viewOn = proto.targets.tail.head,
pipeline = proto.bsonParam.map(p => protoToBson(p))))
)
proto.options match {
case None => ctx
case Some(o) => ctx.setCommand(CreateView(viewName = proto.targets.head,
viewOn = proto.targets.tail.head,
pipeline = proto.bsonParam.map(p => protoToBson(p)),
options = Some(unmarshal[Any](o.value)))
)
}
}
case MGO_ADMIN_CREATEINDEX=> {
var ctx = new MGOContext(
dbName = proto.dbName,
collName = proto.collName,
actionType = MGO_ADMIN,
action = Some(CreateIndex(key = protoToBson(proto.bsonParam.head)))
)
proto.options match {
case None => ctx
case Some(o) => ctx.setCommand(CreateIndex(key = protoToBson(proto.bsonParam.head),
options = Some(unmarshal[Any](o.value)))
)
}
}
case MGO_ADMIN_DROPINDEXBYNAME=> {
var ctx = new MGOContext(
dbName = proto.dbName,
collName = proto.collName,
actionType = MGO_ADMIN,
action = Some(DropIndexByName(indexName = proto.targets.head))
)
proto.options match {
case None => ctx
case Some(o) => ctx.setCommand(DropIndexByName(indexName = proto.targets.head,
options = Some(unmarshal[Any](o.value)))
)
}
}
case MGO_ADMIN_DROPINDEXBYKEY=> {
var ctx = new MGOContext(
dbName = proto.dbName,
collName = proto.collName,
actionType = MGO_ADMIN,
action = Some(DropIndexByKey(key = protoToBson(proto.bsonParam.head)))
)
proto.options match {
case None => ctx
case Some(o) => ctx.setCommand(DropIndexByKey(key = protoToBson(proto.bsonParam.head),
options = Some(unmarshal[Any](o.value)))
)
}
}
case MGO_ADMIN_DROPALLINDEXES=> {
var ctx = new MGOContext(
dbName = proto.dbName,
collName = proto.collName,
actionType = MGO_ADMIN,
action = Some(DropAllIndexes())
)
proto.options match {
case None => ctx
case Some(o) => ctx.setCommand(DropAllIndexes(
options = Some(unmarshal[Any](o.value)))
)
}
} } def ctxToProto(ctx: MGOContext): Option[sdp.grpc.services.ProtoMGOContext] = ctx.action match {
case None => None
case Some(act) => act match {
case Count(filter, options) =>
Some(new sdp.grpc.services.ProtoMGOContext(
dbName = ctx.dbName,
collName = ctx.collName,
commandType = MGO_COMMAND_COUNT,
bsonParam = { if (filter == None) Seq.empty[ProtoMGOBson]
else Seq(bsonToProto(filter.get))},
options = { if(options == None) None //Some(ProtoAny(com.google.protobuf.ByteString.EMPTY))
else Some(ProtoAny(marshal(options.get))) }
))
case Distict(fieldName, filter) =>
Some(new sdp.grpc.services.ProtoMGOContext(
dbName = ctx.dbName,
collName = ctx.collName,
commandType = MGO_COMMAND_DISTICT,
bsonParam = { if (filter == None) Seq.empty[ProtoMGOBson]
else Seq(bsonToProto(filter.get))},
targets = Seq(fieldName) )) case Find(filter, andThen, firstOnly) =>
Some(new sdp.grpc.services.ProtoMGOContext(
dbName = ctx.dbName,
collName = ctx.collName,
commandType = MGO_COMMAND_FIND,
bsonParam = { if (filter == None) Seq.empty[ProtoMGOBson]
else Seq(bsonToProto(filter.get))},
resultOptions = andThen.map(_.toProto)
)) case Aggregate(pipeLine) =>
Some(new sdp.grpc.services.ProtoMGOContext(
dbName = ctx.dbName,
collName = ctx.collName,
commandType = MGO_COMMAND_AGGREGATE,
bsonParam = pipeLine.map(bsonToProto)
)) case Insert(newdocs, options) =>
Some(new sdp.grpc.services.ProtoMGOContext(
dbName = ctx.dbName,
collName = ctx.collName,
commandType = MGO_COMMAND_INSERT,
documents = newdocs.map(d => ProtoMGODocument(marshal(d))),
options = { if(options == None) None //Some(ProtoAny(com.google.protobuf.ByteString.EMPTY))
else Some(ProtoAny(marshal(options.get))) }
)) case Delete(filter, options, onlyOne) =>
Some(new sdp.grpc.services.ProtoMGOContext(
dbName = ctx.dbName,
collName = ctx.collName,
commandType = MGO_COMMAND_DELETE,
bsonParam = Seq(bsonToProto(filter)),
options = { if(options == None) None //Some(ProtoAny(com.google.protobuf.ByteString.EMPTY))
else Some(ProtoAny(marshal(options.get))) },
only = Some(onlyOne)
)) case Replace(filter, replacement, options) =>
Some(new sdp.grpc.services.ProtoMGOContext(
dbName = ctx.dbName,
collName = ctx.collName,
commandType = MGO_COMMAND_REPLACE,
bsonParam = Seq(bsonToProto(filter)),
options = { if(options == None) None //Some(ProtoAny(com.google.protobuf.ByteString.EMPTY))
else Some(ProtoAny(marshal(options.get))) },
documents = Seq(ProtoMGODocument(marshal(replacement)))
)) case Update(filter, update, options, onlyOne) =>
Some(new sdp.grpc.services.ProtoMGOContext(
dbName = ctx.dbName,
collName = ctx.collName,
commandType = MGO_COMMAND_UPDATE,
bsonParam = Seq(bsonToProto(filter),bsonToProto(update)),
options = { if(options == None) None //Some(ProtoAny(com.google.protobuf.ByteString.EMPTY))
else Some(ProtoAny(marshal(options.get))) },
only = Some(onlyOne)
)) case DropCollection(coll) =>
Some(new sdp.grpc.services.ProtoMGOContext(
dbName = ctx.dbName,
collName = coll,
commandType = MGO_ADMIN_DROPCOLLECTION
)) case CreateCollection(coll, options) =>
Some(new sdp.grpc.services.ProtoMGOContext(
dbName = ctx.dbName,
collName = coll,
commandType = MGO_ADMIN_CREATECOLLECTION,
options = { if(options == None) None //Some(ProtoAny(com.google.protobuf.ByteString.EMPTY))
else Some(ProtoAny(marshal(options.get))) }
)) case ListCollection(dbName) =>
Some(new sdp.grpc.services.ProtoMGOContext(
dbName = ctx.dbName,
commandType = MGO_ADMIN_LISTCOLLECTION
)) case CreateView(viewName, viewOn, pipeline, options) =>
Some(new sdp.grpc.services.ProtoMGOContext(
dbName = ctx.dbName,
collName = ctx.collName,
commandType = MGO_ADMIN_CREATEVIEW,
bsonParam = pipeline.map(bsonToProto),
options = { if(options == None) None //Some(ProtoAny(com.google.protobuf.ByteString.EMPTY))
else Some(ProtoAny(marshal(options.get))) },
targets = Seq(viewName,viewOn)
)) case CreateIndex(key, options) =>
Some(new sdp.grpc.services.ProtoMGOContext(
dbName = ctx.dbName,
collName = ctx.collName,
commandType = MGO_ADMIN_CREATEINDEX,
bsonParam = Seq(bsonToProto(key)),
options = { if(options == None) None //Some(ProtoAny(com.google.protobuf.ByteString.EMPTY))
else Some(ProtoAny(marshal(options.get))) }
)) case DropIndexByName(indexName, options) =>
Some(new sdp.grpc.services.ProtoMGOContext(
dbName = ctx.dbName,
collName = ctx.collName,
commandType = MGO_ADMIN_DROPINDEXBYNAME,
targets = Seq(indexName),
options = { if(options == None) None //Some(ProtoAny(com.google.protobuf.ByteString.EMPTY))
else Some(ProtoAny(marshal(options.get))) }
)) case DropIndexByKey(key, options) =>
Some(new sdp.grpc.services.ProtoMGOContext(
dbName = ctx.dbName,
collName = ctx.collName,
commandType = MGO_ADMIN_DROPINDEXBYKEY,
bsonParam = Seq(bsonToProto(key)),
options = { if(options == None) None //Some(ProtoAny(com.google.protobuf.ByteString.EMPTY))
else Some(ProtoAny(marshal(options.get))) }
)) case DropAllIndexes(options) =>
Some(new sdp.grpc.services.ProtoMGOContext(
dbName = ctx.dbName,
collName = ctx.collName,
commandType = MGO_ADMIN_DROPALLINDEXES,
options = { if(options == None) None //Some(ProtoAny(com.google.protobuf.ByteString.EMPTY))
else Some(ProtoAny(marshal(options.get))) }
)) }
} }
mgo/ObservableToPublisher.scala
package sdp.mongo.engine
import java.util.concurrent.atomic.AtomicBoolean
import org.mongodb.{scala => mongoDB}
import org.{reactivestreams => rxStreams}
final case class ObservableToPublisher[T](observable: mongoDB.Observable[T])
extends rxStreams.Publisher[T] {
def subscribe(subscriber: rxStreams.Subscriber[_ >: T]): Unit =
observable.subscribe(new mongoDB.Observer[T]() {
override def onSubscribe(subscription: mongoDB.Subscription): Unit =
subscriber.onSubscribe(new rxStreams.Subscription() {
private final val cancelled: AtomicBoolean = new AtomicBoolean
override def request(n: Long): Unit =
if (!subscription.isUnsubscribed && !cancelled.get() && n < 1) {
subscriber.onError(
new IllegalArgumentException(
s"Demand from publisher should be a positive amount. Current amount is:$n"
)
)
} else {
subscription.request(n)
}
override def cancel(): Unit =
if (!cancelled.getAndSet(true)) subscription.unsubscribe()
})
def onNext(result: T): Unit = subscriber.onNext(result)
def onError(e: Throwable): Unit = subscriber.onError(e)
def onComplete(): Unit = subscriber.onComplete()
})
}
mgo/MongoEngine.scala
package sdp.mongo.engine import java.text.SimpleDateFormat
import java.util.Calendar import akka.NotUsed
import akka.stream.Materializer
import akka.stream.alpakka.mongodb.scaladsl._
import akka.stream.scaladsl.{Flow, Source}
import org.bson.conversions.Bson
import org.mongodb.scala.bson.collection.immutable.Document
import org.mongodb.scala.bson.{BsonArray, BsonBinary}
import org.mongodb.scala.model._
import org.mongodb.scala.{MongoClient, _}
import protobuf.bytes.Converter._
import sdp.file.Streaming._
import sdp.logging.LogSupport import scala.collection.JavaConverters._
import scala.concurrent._
import scala.concurrent.duration._ object MGOClasses {
type MGO_ACTION_TYPE = Int
val MGO_QUERY = 0
val MGO_UPDATE = 1
val MGO_ADMIN = 2 /* org.mongodb.scala.FindObservable
import com.mongodb.async.client.FindIterable
val resultDocType = FindIterable[Document]
val resultOption = FindObservable(resultDocType)
.maxScan(...)
.limit(...)
.sort(...)
.project(...) */ type FOD_TYPE = Int
val FOD_FIRST = 0 //def first(): SingleObservable[TResult], return the first item
val FOD_FILTER = 1 //def filter(filter: Bson): FindObservable[TResult]
val FOD_LIMIT = 2 //def limit(limit: Int): FindObservable[TResult]
val FOD_SKIP = 3 //def skip(skip: Int): FindObservable[TResult]
val FOD_PROJECTION = 4 //def projection(projection: Bson): FindObservable[TResult]
//Sets a document describing the fields to return for all matching documents
val FOD_SORT = 5 //def sort(sort: Bson): FindObservable[TResult]
val FOD_PARTIAL = 6 //def partial(partial: Boolean): FindObservable[TResult]
//Get partial results from a sharded cluster if one or more shards are unreachable (instead of throwing an error)
val FOD_CURSORTYPE = 7 //def cursorType(cursorType: CursorType): FindObservable[TResult]
//Sets the cursor type
val FOD_HINT = 8 //def hint(hint: Bson): FindObservable[TResult]
//Sets the hint for which index to use. A null value means no hint is set
val FOD_MAX = 9 //def max(max: Bson): FindObservable[TResult]
//Sets the exclusive upper bound for a specific index. A null value means no max is set
val FOD_MIN = 10 //def min(min: Bson): FindObservable[TResult]
//Sets the minimum inclusive lower bound for a specific index. A null value means no max is set
val FOD_RETURNKEY = 11 //def returnKey(returnKey: Boolean): FindObservable[TResult]
//Sets the returnKey. If true the find operation will return only the index keys in the resulting documents
val FOD_SHOWRECORDID=12 //def showRecordId(showRecordId: Boolean): FindObservable[TResult]
//Sets the showRecordId. Set to true to add a field `\$recordId` to the returned documents case class ResultOptions(
optType: FOD_TYPE,
bson: Option[Bson] = None,
value: Int = 0 ){
def toProto = new sdp.grpc.services.ProtoMGOResultOption(
optType = this.optType,
bsonParam = this.bson.map {b => sdp.grpc.services.ProtoMGOBson(marshal(b))},
valueParam = this.value
)
def toFindObservable: FindObservable[Document] => FindObservable[Document] = find => {
optType match {
case FOD_FIRST => find
case FOD_FILTER => find.filter(bson.get)
case FOD_LIMIT => find.limit(value)
case FOD_SKIP => find.skip(value)
case FOD_PROJECTION => find.projection(bson.get)
case FOD_SORT => find.sort(bson.get)
case FOD_PARTIAL => find.partial(value != 0)
case FOD_CURSORTYPE => find
case FOD_HINT => find.hint(bson.get)
case FOD_MAX => find.max(bson.get)
case FOD_MIN => find.min(bson.get)
case FOD_RETURNKEY => find.returnKey(value != 0)
case FOD_SHOWRECORDID => find.showRecordId(value != 0) }
}
}
object ResultOptions {
def fromProto(msg: sdp.grpc.services.ProtoMGOResultOption) = new ResultOptions(
optType = msg.optType,
bson = msg.bsonParam.map(b => unmarshal[Bson](b.bson)),
value = msg.valueParam
) } trait MGOCommands object MGOCommands { case class Count(filter: Option[Bson] = None, options: Option[Any] = None) extends MGOCommands case class Distict(fieldName: String, filter: Option[Bson] = None) extends MGOCommands /* org.mongodb.scala.FindObservable
import com.mongodb.async.client.FindIterable
val resultDocType = FindIterable[Document]
val resultOption = FindObservable(resultDocType)
.maxScan(...)
.limit(...)
.sort(...)
.project(...) */
case class Find(filter: Option[Bson] = None,
andThen: Seq[ResultOptions] = Seq.empty[ResultOptions],
firstOnly: Boolean = false) extends MGOCommands case class Aggregate(pipeLine: Seq[Bson]) extends MGOCommands case class MapReduce(mapFunction: String, reduceFunction: String) extends MGOCommands case class Insert(newdocs: Seq[Document], options: Option[Any] = None) extends MGOCommands case class Delete(filter: Bson, options: Option[Any] = None, onlyOne: Boolean = false) extends MGOCommands case class Replace(filter: Bson, replacement: Document, options: Option[Any] = None) extends MGOCommands case class Update(filter: Bson, update: Bson, options: Option[Any] = None, onlyOne: Boolean = false) extends MGOCommands case class BulkWrite(commands: List[WriteModel[Document]], options: Option[Any] = None) extends MGOCommands } object MGOAdmins { case class DropCollection(collName: String) extends MGOCommands case class CreateCollection(collName: String, options: Option[Any] = None) extends MGOCommands case class ListCollection(dbName: String) extends MGOCommands case class CreateView(viewName: String, viewOn: String, pipeline: Seq[Bson], options: Option[Any] = None) extends MGOCommands case class CreateIndex(key: Bson, options: Option[Any] = None) extends MGOCommands case class DropIndexByName(indexName: String, options: Option[Any] = None) extends MGOCommands case class DropIndexByKey(key: Bson, options: Option[Any] = None) extends MGOCommands case class DropAllIndexes(options: Option[Any] = None) extends MGOCommands } case class MGOContext(
dbName: String,
collName: String,
actionType: MGO_ACTION_TYPE = MGO_QUERY,
action: Option[MGOCommands] = None,
actionOptions: Option[Any] = None,
actionTargets: Seq[String] = Nil
) {
ctx =>
def setDbName(name: String): MGOContext = ctx.copy(dbName = name) def setCollName(name: String): MGOContext = ctx.copy(collName = name) def setActionType(at: MGO_ACTION_TYPE): MGOContext = ctx.copy(actionType = at) def setCommand(cmd: MGOCommands): MGOContext = ctx.copy(action = Some(cmd)) def toSomeProto = MGOProtoConversion.ctxToProto(this) } object MGOContext {
def apply(db: String, coll: String) = new MGOContext(db, coll)
def fromProto(proto: sdp.grpc.services.ProtoMGOContext): MGOContext =
MGOProtoConversion.ctxFromProto(proto)
} case class MGOBatContext(contexts: Seq[MGOContext], tx: Boolean = false) {
ctxs =>
def setTx(txopt: Boolean): MGOBatContext = ctxs.copy(tx = txopt)
def appendContext(ctx: MGOContext): MGOBatContext =
ctxs.copy(contexts = contexts :+ ctx)
} object MGOBatContext {
def apply(ctxs: Seq[MGOContext], tx: Boolean = false) = new MGOBatContext(ctxs,tx)
} type MGODate = java.util.Date
def mgoDate(yyyy: Int, mm: Int, dd: Int): MGODate = {
val ca = Calendar.getInstance()
ca.set(yyyy,mm,dd)
ca.getTime()
}
def mgoDateTime(yyyy: Int, mm: Int, dd: Int, hr: Int, min: Int, sec: Int): MGODate = {
val ca = Calendar.getInstance()
ca.set(yyyy,mm,dd,hr,min,sec)
ca.getTime()
}
def mgoDateTimeNow: MGODate = {
val ca = Calendar.getInstance()
ca.getTime
} def mgoDateToString(dt: MGODate, formatString: String): String = {
val fmt= new SimpleDateFormat(formatString)
fmt.format(dt)
} type MGOBlob = BsonBinary
type MGOArray = BsonArray def fileToMGOBlob(fileName: String, timeOut: FiniteDuration = 60 seconds)(
implicit mat: Materializer) = FileToByteArray(fileName,timeOut) def mgoBlobToFile(blob: MGOBlob, fileName: String)(
implicit mat: Materializer) = ByteArrayToFile(blob.getData,fileName) def mgoGetStringOrNone(doc: Document, fieldName: String) = {
if (doc.keySet.contains(fieldName))
Some(doc.getString(fieldName))
else None
}
def mgoGetIntOrNone(doc: Document, fieldName: String) = {
if (doc.keySet.contains(fieldName))
Some(doc.getInteger(fieldName))
else None
}
def mgoGetLonggOrNone(doc: Document, fieldName: String) = {
if (doc.keySet.contains(fieldName))
Some(doc.getLong(fieldName))
else None
}
def mgoGetDoubleOrNone(doc: Document, fieldName: String) = {
if (doc.keySet.contains(fieldName))
Some(doc.getDouble(fieldName))
else None
}
def mgoGetBoolOrNone(doc: Document, fieldName: String) = {
if (doc.keySet.contains(fieldName))
Some(doc.getBoolean(fieldName))
else None
}
def mgoGetDateOrNone(doc: Document, fieldName: String) = {
if (doc.keySet.contains(fieldName))
Some(doc.getDate(fieldName))
else None
}
def mgoGetBlobOrNone(doc: Document, fieldName: String) = {
if (doc.keySet.contains(fieldName))
doc.get(fieldName).asInstanceOf[Option[MGOBlob]]
else None
}
def mgoGetArrayOrNone(doc: Document, fieldName: String) = {
if (doc.keySet.contains(fieldName))
doc.get(fieldName).asInstanceOf[Option[MGOArray]]
else None
} def mgoArrayToDocumentList(arr: MGOArray): scala.collection.immutable.List[org.bson.BsonDocument] = {
(arr.getValues.asScala.toList)
.asInstanceOf[scala.collection.immutable.List[org.bson.BsonDocument]]
} type MGOFilterResult = FindObservable[Document] => FindObservable[Document]
} object MGOEngine extends LogSupport { import MGOClasses._
import MGOAdmins._
import MGOCommands._
import sdp.result.DBOResult._
import com.mongodb.reactivestreams.client.MongoClients object TxUpdateMode {
private def mgoTxUpdate(ctxs: MGOBatContext, observable: SingleObservable[ClientSession])(
implicit client: MongoClient, ec: ExecutionContext): SingleObservable[ClientSession] = {
log.info(s"mgoTxUpdate> calling ...")
observable.map(clientSession => { val transactionOptions =
TransactionOptions.builder()
.readConcern(ReadConcern.SNAPSHOT)
.writeConcern(WriteConcern.MAJORITY).build() clientSession.startTransaction(transactionOptions)
/*
val fut = Future.traverse(ctxs.contexts) { ctx =>
mgoUpdateObservable[Completed](ctx).map(identity).toFuture()
}
Await.ready(fut, 3 seconds) */ ctxs.contexts.foreach { ctx =>
mgoUpdateObservable[Completed](ctx).map(identity).toFuture()
}
clientSession
})
} private def commitAndRetry(observable: SingleObservable[Completed]): SingleObservable[Completed] = {
log.info(s"commitAndRetry> calling ...")
observable.recoverWith({
case e: MongoException if e.hasErrorLabel(MongoException.UNKNOWN_TRANSACTION_COMMIT_RESULT_LABEL) => {
log.warn("commitAndRetry> UnknownTransactionCommitResult, retrying commit operation ...")
commitAndRetry(observable)
}
case e: Exception => {
log.error(s"commitAndRetry> Exception during commit ...: $e")
throw e
}
})
} private def runTransactionAndRetry(observable: SingleObservable[Completed]): SingleObservable[Completed] = {
log.info(s"runTransactionAndRetry> calling ...")
observable.recoverWith({
case e: MongoException if e.hasErrorLabel(MongoException.TRANSIENT_TRANSACTION_ERROR_LABEL) => {
log.warn("runTransactionAndRetry> TransientTransactionError, aborting transaction and retrying ...")
runTransactionAndRetry(observable)
}
})
} def mgoTxBatch(ctxs: MGOBatContext)(
implicit client: MongoClient, ec: ExecutionContext): DBOResult[Completed] = { log.info(s"mgoTxBatch> MGOBatContext: ${ctxs}") val updateObservable: Observable[ClientSession] = mgoTxUpdate(ctxs, client.startSession())
val commitTransactionObservable: SingleObservable[Completed] =
updateObservable.flatMap(clientSession => clientSession.commitTransaction())
val commitAndRetryObservable: SingleObservable[Completed] = commitAndRetry(commitTransactionObservable) runTransactionAndRetry(commitAndRetryObservable) valueToDBOResult(Completed()) }
} def mgoUpdateBatch(ctxs: MGOBatContext)(implicit client: MongoClient, ec: ExecutionContext): DBOResult[Completed] = {
log.info(s"mgoUpdateBatch> MGOBatContext: ${ctxs}")
if (ctxs.tx) {
TxUpdateMode.mgoTxBatch(ctxs)
} else {
/*
val fut = Future.traverse(ctxs.contexts) { ctx =>
mgoUpdate[Completed](ctx).map(identity) } Await.ready(fut, 3 seconds)
Future.successful(new Completed) */
ctxs.contexts.foreach { ctx =>
mgoUpdate[Completed](ctx).map(identity) } valueToDBOResult(Completed())
} } def mongoStream(ctx: MGOContext)(
implicit client: MongoClient, ec: ExecutionContextExecutor): Source[Document, NotUsed] = { log.info(s"mongoStream> MGOContext: ${ctx}") ObservableToPublisher def toResultOption(rts: Seq[ResultOptions]): FindObservable[Document] => FindObservable[Document] = findObj =>
rts.foldRight(findObj)((a,b) => a.toFindObservable(b)) val db = client.getDatabase(ctx.dbName)
val coll = db.getCollection(ctx.collName)
if ( ctx.action == None) {
log.error(s"mongoStream> uery action cannot be null!")
throw new IllegalArgumentException("query action cannot be null!")
}
try {
ctx.action.get match {
case Find(None, Nil, false) => //FindObservable
MongoSource(ObservableToPublisher(coll.find()))
case Find(None, Nil, true) => //FindObservable
MongoSource(ObservableToPublisher(coll.find().first()))
case Find(Some(filter), Nil, false) => //FindObservable
MongoSource(ObservableToPublisher(coll.find(filter)))
case Find(Some(filter), Nil, true) => //FindObservable
MongoSource(ObservableToPublisher(coll.find(filter).first()))
case Find(None, sro, _) => //FindObservable
val next = toResultOption(sro)
MongoSource(ObservableToPublisher(next(coll.find[Document]())))
case Find(Some(filter), sro, _) => //FindObservable
val next = toResultOption(sro)
MongoSource(ObservableToPublisher(next(coll.find[Document](filter))))
case _ =>
log.error(s"mongoStream> unsupported streaming query [${ctx.action.get}]")
throw new RuntimeException(s"mongoStream> unsupported streaming query [${ctx.action.get}]") }
}
catch { case e: Exception =>
log.error(s"mongoStream> runtime error: ${e.getMessage}")
throw new RuntimeException(s"mongoStream> Error: ${e.getMessage}")
} } // T => FindIterable e.g List[Document]
def mgoQuery[T](ctx: MGOContext, Converter: Option[Document => Any] = None)(implicit client: MongoClient): DBOResult[T] = {
log.info(s"mgoQuery> MGOContext: ${ctx}") val db = client.getDatabase(ctx.dbName)
val coll = db.getCollection(ctx.collName) def toResultOption(rts: Seq[ResultOptions]): FindObservable[Document] => FindObservable[Document] = findObj =>
rts.foldRight(findObj)((a,b) => a.toFindObservable(b)) if ( ctx.action == None) {
log.error(s"mgoQuery> uery action cannot be null!")
Left(new IllegalArgumentException("query action cannot be null!"))
}
try {
ctx.action.get match {
/* count */
case Count(Some(filter), Some(opt)) => //SingleObservable
coll.countDocuments(filter, opt.asInstanceOf[CountOptions])
.toFuture().asInstanceOf[Future[T]]
case Count(Some(filter), None) => //SingleObservable
coll.countDocuments(filter).toFuture()
.asInstanceOf[Future[T]]
case Count(None, None) => //SingleObservable
coll.countDocuments().toFuture()
.asInstanceOf[Future[T]]
/* distinct */
case Distict(field, Some(filter)) => //DistinctObservable
coll.distinct(field, filter).toFuture()
.asInstanceOf[Future[T]]
case Distict(field, None) => //DistinctObservable
coll.distinct((field)).toFuture()
.asInstanceOf[Future[T]]
/* find */
case Find(None, Nil, false) => //FindObservable
if (Converter == None) coll.find().toFuture().asInstanceOf[Future[T]]
else coll.find().map(Converter.get).toFuture().asInstanceOf[Future[T]]
case Find(None, Nil, true) => //FindObservable
if (Converter == None) coll.find().first().head().asInstanceOf[Future[T]]
else coll.find().first().map(Converter.get).head().asInstanceOf[Future[T]]
case Find(Some(filter), Nil, false) => //FindObservable
if (Converter == None) coll.find(filter).toFuture().asInstanceOf[Future[T]]
else coll.find(filter).map(Converter.get).toFuture().asInstanceOf[Future[T]]
case Find(Some(filter), Nil, true) => //FindObservable
if (Converter == None) coll.find(filter).first().head().asInstanceOf[Future[T]]
else coll.find(filter).first().map(Converter.get).head().asInstanceOf[Future[T]]
case Find(None, sro, _) => //FindObservable
val next = toResultOption(sro)
if (Converter == None) next(coll.find[Document]()).toFuture().asInstanceOf[Future[T]]
else next(coll.find[Document]()).map(Converter.get).toFuture().asInstanceOf[Future[T]]
case Find(Some(filter), sro, _) => //FindObservable
val next = toResultOption(sro)
if (Converter == None) next(coll.find[Document](filter)).toFuture().asInstanceOf[Future[T]]
else next(coll.find[Document](filter)).map(Converter.get).toFuture().asInstanceOf[Future[T]]
/* aggregate AggregateObservable*/
case Aggregate(pline) => coll.aggregate(pline).toFuture().asInstanceOf[Future[T]]
/* mapReduce MapReduceObservable*/
case MapReduce(mf, rf) => coll.mapReduce(mf, rf).toFuture().asInstanceOf[Future[T]]
/* list collection */
case ListCollection(dbName) => //ListConllectionObservable
client.getDatabase(dbName).listCollections().toFuture().asInstanceOf[Future[T]] }
}
catch { case e: Exception =>
log.error(s"mgoQuery> runtime error: ${e.getMessage}")
Left(new RuntimeException(s"mgoQuery> Error: ${e.getMessage}"))
}
}
//T => Completed, result.UpdateResult, result.DeleteResult
def mgoUpdate[T](ctx: MGOContext)(implicit client: MongoClient): DBOResult[T] =
try {
mgoUpdateObservable[T](ctx).toFuture()
}
catch { case e: Exception =>
log.error(s"mgoUpdate> runtime error: ${e.getMessage}")
Left(new RuntimeException(s"mgoUpdate> Error: ${e.getMessage}"))
} def mgoUpdateObservable[T](ctx: MGOContext)(implicit client: MongoClient): SingleObservable[T] = {
log.info(s"mgoUpdateObservable> MGOContext: ${ctx}") val db = client.getDatabase(ctx.dbName)
val coll = db.getCollection(ctx.collName)
if ( ctx.action == None) {
log.error(s"mgoUpdateObservable> uery action cannot be null!")
throw new IllegalArgumentException("mgoUpdateObservable> query action cannot be null!")
}
try {
ctx.action.get match {
/* insert */
case Insert(docs, Some(opt)) => //SingleObservable[Completed]
if (docs.size > 1)
coll.insertMany(docs, opt.asInstanceOf[InsertManyOptions]).asInstanceOf[SingleObservable[T]]
else coll.insertOne(docs.head, opt.asInstanceOf[InsertOneOptions]).asInstanceOf[SingleObservable[T]]
case Insert(docs, None) => //SingleObservable
if (docs.size > 1) coll.insertMany(docs).asInstanceOf[SingleObservable[T]]
else coll.insertOne(docs.head).asInstanceOf[SingleObservable[T]]
/* delete */
case Delete(filter, None, onlyOne) => //SingleObservable
if (onlyOne) coll.deleteOne(filter).asInstanceOf[SingleObservable[T]]
else coll.deleteMany(filter).asInstanceOf[SingleObservable[T]]
case Delete(filter, Some(opt), onlyOne) => //SingleObservable
if (onlyOne) coll.deleteOne(filter, opt.asInstanceOf[DeleteOptions]).asInstanceOf[SingleObservable[T]]
else coll.deleteMany(filter, opt.asInstanceOf[DeleteOptions]).asInstanceOf[SingleObservable[T]]
/* replace */
case Replace(filter, replacement, None) => //SingleObservable
coll.replaceOne(filter, replacement).asInstanceOf[SingleObservable[T]]
case Replace(filter, replacement, Some(opt)) => //SingleObservable
coll.replaceOne(filter, replacement, opt.asInstanceOf[ReplaceOptions]).asInstanceOf[SingleObservable[T]]
/* update */
case Update(filter, update, None, onlyOne) => //SingleObservable
if (onlyOne) coll.updateOne(filter, update).asInstanceOf[SingleObservable[T]]
else coll.updateMany(filter, update).asInstanceOf[SingleObservable[T]]
case Update(filter, update, Some(opt), onlyOne) => //SingleObservable
if (onlyOne) coll.updateOne(filter, update, opt.asInstanceOf[UpdateOptions]).asInstanceOf[SingleObservable[T]]
else coll.updateMany(filter, update, opt.asInstanceOf[UpdateOptions]).asInstanceOf[SingleObservable[T]]
/* bulkWrite */
case BulkWrite(commands, None) => //SingleObservable
coll.bulkWrite(commands).asInstanceOf[SingleObservable[T]]
case BulkWrite(commands, Some(opt)) => //SingleObservable
coll.bulkWrite(commands, opt.asInstanceOf[BulkWriteOptions]).asInstanceOf[SingleObservable[T]]
}
}
catch { case e: Exception =>
log.error(s"mgoUpdateObservable> runtime error: ${e.getMessage}")
throw new RuntimeException(s"mgoUpdateObservable> Error: ${e.getMessage}")
}
} def mgoAdmin(ctx: MGOContext)(implicit client: MongoClient): DBOResult[Completed] = {
log.info(s"mgoAdmin> MGOContext: ${ctx}") val db = client.getDatabase(ctx.dbName)
val coll = db.getCollection(ctx.collName)
if ( ctx.action == None) {
log.error(s"mgoAdmin> uery action cannot be null!")
Left(new IllegalArgumentException("mgoAdmin> query action cannot be null!"))
}
try {
ctx.action.get match {
/* drop collection */
case DropCollection(collName) => //SingleObservable
val coll = db.getCollection(collName)
coll.drop().toFuture()
/* create collection */
case CreateCollection(collName, None) => //SingleObservable
db.createCollection(collName).toFuture()
case CreateCollection(collName, Some(opt)) => //SingleObservable
db.createCollection(collName, opt.asInstanceOf[CreateCollectionOptions]).toFuture()
/* list collection
case ListCollection(dbName) => //ListConllectionObservable
client.getDatabase(dbName).listCollections().toFuture().asInstanceOf[Future[T]]
*/
/* create view */
case CreateView(viewName, viewOn, pline, None) => //SingleObservable
db.createView(viewName, viewOn, pline).toFuture()
case CreateView(viewName, viewOn, pline, Some(opt)) => //SingleObservable
db.createView(viewName, viewOn, pline, opt.asInstanceOf[CreateViewOptions]).toFuture()
/* create index */
case CreateIndex(key, None) => //SingleObservable
coll.createIndex(key).toFuture().asInstanceOf[Future[Completed]] // asInstanceOf[SingleObservable[Completed]]
case CreateIndex(key, Some(opt)) => //SingleObservable
coll.createIndex(key, opt.asInstanceOf[IndexOptions]).asInstanceOf[Future[Completed]] // asInstanceOf[SingleObservable[Completed]]
/* drop index */
case DropIndexByName(indexName, None) => //SingleObservable
coll.dropIndex(indexName).toFuture()
case DropIndexByName(indexName, Some(opt)) => //SingleObservable
coll.dropIndex(indexName, opt.asInstanceOf[DropIndexOptions]).toFuture()
case DropIndexByKey(key, None) => //SingleObservable
coll.dropIndex(key).toFuture()
case DropIndexByKey(key, Some(opt)) => //SingleObservable
coll.dropIndex(key, opt.asInstanceOf[DropIndexOptions]).toFuture()
case DropAllIndexes(None) => //SingleObservable
coll.dropIndexes().toFuture()
case DropAllIndexes(Some(opt)) => //SingleObservable
coll.dropIndexes(opt.asInstanceOf[DropIndexOptions]).toFuture()
}
}
catch { case e: Exception =>
log.error(s"mgoAdmin> runtime error: ${e.getMessage}")
throw new RuntimeException(s"mgoAdmin> Error: ${e.getMessage}")
} } /*
def mgoExecute[T](ctx: MGOContext)(implicit client: MongoClient): Future[T] = {
val db = client.getDatabase(ctx.dbName)
val coll = db.getCollection(ctx.collName)
ctx.action match {
/* count */
case Count(Some(filter), Some(opt)) => //SingleObservable
coll.countDocuments(filter, opt.asInstanceOf[CountOptions])
.toFuture().asInstanceOf[Future[T]]
case Count(Some(filter), None) => //SingleObservable
coll.countDocuments(filter).toFuture()
.asInstanceOf[Future[T]]
case Count(None, None) => //SingleObservable
coll.countDocuments().toFuture()
.asInstanceOf[Future[T]]
/* distinct */
case Distict(field, Some(filter)) => //DistinctObservable
coll.distinct(field, filter).toFuture()
.asInstanceOf[Future[T]]
case Distict(field, None) => //DistinctObservable
coll.distinct((field)).toFuture()
.asInstanceOf[Future[T]]
/* find */
case Find(None, None, optConv, false) => //FindObservable
if (optConv == None) coll.find().toFuture().asInstanceOf[Future[T]]
else coll.find().map(optConv.get).toFuture().asInstanceOf[Future[T]]
case Find(None, None, optConv, true) => //FindObservable
if (optConv == None) coll.find().first().head().asInstanceOf[Future[T]]
else coll.find().first().map(optConv.get).head().asInstanceOf[Future[T]]
case Find(Some(filter), None, optConv, false) => //FindObservable
if (optConv == None) coll.find(filter).toFuture().asInstanceOf[Future[T]]
else coll.find(filter).map(optConv.get).toFuture().asInstanceOf[Future[T]]
case Find(Some(filter), None, optConv, true) => //FindObservable
if (optConv == None) coll.find(filter).first().head().asInstanceOf[Future[T]]
else coll.find(filter).first().map(optConv.get).head().asInstanceOf[Future[T]]
case Find(None, Some(next), optConv, _) => //FindObservable
if (optConv == None) next(coll.find[Document]()).toFuture().asInstanceOf[Future[T]]
else next(coll.find[Document]()).map(optConv.get).toFuture().asInstanceOf[Future[T]]
case Find(Some(filter), Some(next), optConv, _) => //FindObservable
if (optConv == None) next(coll.find[Document](filter)).toFuture().asInstanceOf[Future[T]]
else next(coll.find[Document](filter)).map(optConv.get).toFuture().asInstanceOf[Future[T]]
/* aggregate AggregateObservable*/
case Aggregate(pline) => coll.aggregate(pline).toFuture().asInstanceOf[Future[T]]
/* mapReduce MapReduceObservable*/
case MapReduce(mf, rf) => coll.mapReduce(mf, rf).toFuture().asInstanceOf[Future[T]]
/* insert */
case Insert(docs, Some(opt)) => //SingleObservable[Completed]
if (docs.size > 1) coll.insertMany(docs, opt.asInstanceOf[InsertManyOptions]).toFuture()
.asInstanceOf[Future[T]]
else coll.insertOne(docs.head, opt.asInstanceOf[InsertOneOptions]).toFuture()
.asInstanceOf[Future[T]]
case Insert(docs, None) => //SingleObservable
if (docs.size > 1) coll.insertMany(docs).toFuture().asInstanceOf[Future[T]]
else coll.insertOne(docs.head).toFuture().asInstanceOf[Future[T]]
/* delete */
case Delete(filter, None, onlyOne) => //SingleObservable
if (onlyOne) coll.deleteOne(filter).toFuture().asInstanceOf[Future[T]]
else coll.deleteMany(filter).toFuture().asInstanceOf[Future[T]]
case Delete(filter, Some(opt), onlyOne) => //SingleObservable
if (onlyOne) coll.deleteOne(filter, opt.asInstanceOf[DeleteOptions]).toFuture().asInstanceOf[Future[T]]
else coll.deleteMany(filter, opt.asInstanceOf[DeleteOptions]).toFuture().asInstanceOf[Future[T]]
/* replace */
case Replace(filter, replacement, None) => //SingleObservable
coll.replaceOne(filter, replacement).toFuture().asInstanceOf[Future[T]]
case Replace(filter, replacement, Some(opt)) => //SingleObservable
coll.replaceOne(filter, replacement, opt.asInstanceOf[UpdateOptions]).toFuture().asInstanceOf[Future[T]]
/* update */
case Update(filter, update, None, onlyOne) => //SingleObservable
if (onlyOne) coll.updateOne(filter, update).toFuture().asInstanceOf[Future[T]]
else coll.updateMany(filter, update).toFuture().asInstanceOf[Future[T]]
case Update(filter, update, Some(opt), onlyOne) => //SingleObservable
if (onlyOne) coll.updateOne(filter, update, opt.asInstanceOf[UpdateOptions]).toFuture().asInstanceOf[Future[T]]
else coll.updateMany(filter, update, opt.asInstanceOf[UpdateOptions]).toFuture().asInstanceOf[Future[T]]
/* bulkWrite */
case BulkWrite(commands, None) => //SingleObservable
coll.bulkWrite(commands).toFuture().asInstanceOf[Future[T]]
case BulkWrite(commands, Some(opt)) => //SingleObservable
coll.bulkWrite(commands, opt.asInstanceOf[BulkWriteOptions]).toFuture().asInstanceOf[Future[T]] /* drop collection */
case DropCollection(collName) => //SingleObservable
val coll = db.getCollection(collName)
coll.drop().toFuture().asInstanceOf[Future[T]]
/* create collection */
case CreateCollection(collName, None) => //SingleObservable
db.createCollection(collName).toFuture().asInstanceOf[Future[T]]
case CreateCollection(collName, Some(opt)) => //SingleObservable
db.createCollection(collName, opt.asInstanceOf[CreateCollectionOptions]).toFuture().asInstanceOf[Future[T]]
/* list collection */
case ListCollection(dbName) => //ListConllectionObservable
client.getDatabase(dbName).listCollections().toFuture().asInstanceOf[Future[T]]
/* create view */
case CreateView(viewName, viewOn, pline, None) => //SingleObservable
db.createView(viewName, viewOn, pline).toFuture().asInstanceOf[Future[T]]
case CreateView(viewName, viewOn, pline, Some(opt)) => //SingleObservable
db.createView(viewName, viewOn, pline, opt.asInstanceOf[CreateViewOptions]).toFuture().asInstanceOf[Future[T]]
/* create index */
case CreateIndex(key, None) => //SingleObservable
coll.createIndex(key).toFuture().asInstanceOf[Future[T]]
case CreateIndex(key, Some(opt)) => //SingleObservable
coll.createIndex(key, opt.asInstanceOf[IndexOptions]).toFuture().asInstanceOf[Future[T]]
/* drop index */
case DropIndexByName(indexName, None) => //SingleObservable
coll.dropIndex(indexName).toFuture().asInstanceOf[Future[T]]
case DropIndexByName(indexName, Some(opt)) => //SingleObservable
coll.dropIndex(indexName, opt.asInstanceOf[DropIndexOptions]).toFuture().asInstanceOf[Future[T]]
case DropIndexByKey(key, None) => //SingleObservable
coll.dropIndex(key).toFuture().asInstanceOf[Future[T]]
case DropIndexByKey(key, Some(opt)) => //SingleObservable
coll.dropIndex(key, opt.asInstanceOf[DropIndexOptions]).toFuture().asInstanceOf[Future[T]]
case DropAllIndexes(None) => //SingleObservable
coll.dropIndexes().toFuture().asInstanceOf[Future[T]]
case DropAllIndexes(Some(opt)) => //SingleObservable
coll.dropIndexes(opt.asInstanceOf[DropIndexOptions]).toFuture().asInstanceOf[Future[T]]
}
}
*/ } object MongoActionStream { import MGOClasses._ case class StreamingInsert[A](dbName: String,
collName: String,
converter: A => Document,
parallelism: Int = 1
) extends MGOCommands case class StreamingDelete[A](dbName: String,
collName: String,
toFilter: A => Bson,
parallelism: Int = 1,
justOne: Boolean = false
) extends MGOCommands case class StreamingUpdate[A](dbName: String,
collName: String,
toFilter: A => Bson,
toUpdate: A => Bson,
parallelism: Int = 1,
justOne: Boolean = false
) extends MGOCommands case class InsertAction[A](ctx: StreamingInsert[A])(
implicit mongoClient: MongoClient) { val database = mongoClient.getDatabase(ctx.dbName)
val collection = database.getCollection(ctx.collName) def performOnRow(implicit ec: ExecutionContext): Flow[A, Document, NotUsed] =
Flow[A].map(ctx.converter)
.mapAsync(ctx.parallelism)(doc => collection.insertOne(doc).toFuture().map(_ => doc))
} case class UpdateAction[A](ctx: StreamingUpdate[A])(
implicit mongoClient: MongoClient) { val database = mongoClient.getDatabase(ctx.dbName)
val collection = database.getCollection(ctx.collName) def performOnRow(implicit ec: ExecutionContext): Flow[A, A, NotUsed] =
if (ctx.justOne) {
Flow[A]
.mapAsync(ctx.parallelism)(a =>
collection.updateOne(ctx.toFilter(a), ctx.toUpdate(a)).toFuture().map(_ => a))
} else
Flow[A]
.mapAsync(ctx.parallelism)(a =>
collection.updateMany(ctx.toFilter(a), ctx.toUpdate(a)).toFuture().map(_ => a))
} case class DeleteAction[A](ctx: StreamingDelete[A])(
implicit mongoClient: MongoClient) { val database = mongoClient.getDatabase(ctx.dbName)
val collection = database.getCollection(ctx.collName) def performOnRow(implicit ec: ExecutionContext): Flow[A, A, NotUsed] =
if (ctx.justOne) {
Flow[A]
.mapAsync(ctx.parallelism)(a =>
collection.deleteOne(ctx.toFilter(a)).toFuture().map(_ => a))
} else
Flow[A]
.mapAsync(ctx.parallelism)(a =>
collection.deleteMany(ctx.toFilter(a)).toFuture().map(_ => a))
} } object MGOHelpers { implicit class DocumentObservable[C](val observable: Observable[Document]) extends ImplicitObservable[Document] {
override val converter: (Document) => String = (doc) => doc.toJson
} implicit class GenericObservable[C](val observable: Observable[C]) extends ImplicitObservable[C] {
override val converter: (C) => String = (doc) => doc.toString
} trait ImplicitObservable[C] {
val observable: Observable[C]
val converter: (C) => String def results(): Seq[C] = Await.result(observable.toFuture(), 10 seconds) def headResult() = Await.result(observable.head(), 10 seconds) def printResults(initial: String = ""): Unit = {
if (initial.length > 0) print(initial)
results().foreach(res => println(converter(res)))
} def printHeadResult(initial: String = ""): Unit = println(s"${initial}${converter(headResult())}")
} def getResult[T](fut: Future[T], timeOut: Duration = 1 second): T = {
Await.result(fut, timeOut)
} def getResults[T](fut: Future[Iterable[T]], timeOut: Duration = 1 second): Iterable[T] = {
Await.result(fut, timeOut)
} import monix.eval.Task
import monix.execution.Scheduler.Implicits.global final class FutureToTask[A](x: => Future[A]) {
def asTask: Task[A] = Task.deferFuture[A](x)
} final class TaskToFuture[A](x: => Task[A]) {
def asFuture: Future[A] = x.runToFuture
} }
Akka-CQRS（8）- CQRS Reader Actor 应用实例的更多相关文章
- Akka-CQRS（7）- CQRS Reader Actor 示范
我们在这篇通过一个具体CQRS-Reader-Actor的例子来示范akka-persistence的query端编程和应用.在前面的博客里我们设计了一个CQRS模式POS机程序的操作动作录入过程,并 ...
- Akka（2）：Actor生命周期管理 - 监控和监视
在开始讨论Akka中对Actor的生命周期管理前,我们先探讨一下所谓的Actor编程模式.对比起我们习惯的行令式(imperative)编程模式,Actor编程模式更接近现实中的应用场景和功能测试模式 ...
- 以Akka为示例，介绍Actor模型
许多开发者在创建和维护多线程应用程序时经历过各种各样的问题,他们希望能在一个更高层次的抽象上进行工作,以避免直接和线程与锁打交道.为了帮助这些开发者,Arun Manivannan编写了一系列的博客帖 ...
- Akka（1）：Actor - 靠消息驱动的运算器
Akka是由各种角色和功能的Actor组成的,工作的主要原理是把一项大的计算任务分割成小环节,再按各环节的要求构建相应功能的Actor,然后把各环节的运算托付给相应的Actor去独立完成.Akka是个 ...
- CQRS学习——Cqrs补丁，async实验以及实现[其二]
实验——async什么时候提高吞吐 async是一个语法糖,用来简化异步编程,主要是让异步编程在书写上接近于同步编程.总的来收,在await的时候,相当于附加上了一个.ContinueWith(). ...
- Akka（3）： Actor监管 - 细述BackoffSupervisor
在上一篇讨论中我们谈到了监管:在Akka中就是一种直属父子监管树结构,父级Actor负责处理直属子级Actor产生的异常.当时我们把BackoffSupervisor作为父子监管方式的其中一种.实际上 ...
- Akka系列（八）：Akka persistence设计理念之CQRS
前言........ 这一篇文章主要是讲解Akka persistence的核心设计理念,也是CQRS(Command Query Responsibility Segregation)架构设计的典型 ...
- CQRS(Command and Query Responsibility Segregation)与EventSources实例
CQRS The CQRS pattern and event sourcing are not mere simplistic solutions to the problems associate ...
- 浅谈命令查询职责分离(CQRS)模式
在常用的三层架构中,通常都是通过数据访问层来修改或者查询数据,一般修改和查询使用的是相同的实体.在一些业务逻辑简单的系统中可能没有什么问题,但是随着系统逻辑变得复杂,用户增多,这种设计就会出现一些性能 ...
随机推荐
- js对象常用属性和方法：复制一个对象，获取一个对象的所有key和所有value的方法
记录对象的一些实用使用方法及属性 // Object.assign() 多个对象合并 key相同则后面的覆盖前面的 const target = { a: 1, b: 2 }; const sourc ...
- Gin-Go学习笔记二：Gin-Web框架
Gin-Web框架 1> 首先声明,这个是我自己搭建的纯Gin-Web框架,其中有借鉴学习别的想法和代码.已上传到GitHub上.地址为: https://github.com/weiy ...
- 【前端_js】javascript中数组的map()方法
数组的map()方法用于遍历数组,每遍历一个元素就调用回调方法一次,并将回调函数的返回结果作为新数组的元素,被遍历的数组不会被改变. 语法:let newAarray = arr.map(functi ...
- Zabbix 添加内存告警
1.在Template OS Linux模板创建item Configuration-->Templates-->Template OS Linux-->items-->cre ...
- 并发编程（六）--进程/线程池、协程、gevent第三方库
一.进程/线程池 1.进程池 (1)什么是进程池 如果需要创建的子进程数量不大,可以直接利用multiprocess中的Process来创建.但是当需要创建上百个或上千个,手动创建就较为繁琐,这时就可 ...
- 源码解析 || ArrayList源码解析
前言 这篇文章的ArrayList源码是基于jdk1.8版本的源码,如果与前后版本的实现细节出现不一致的地方请自己多加注意.先上一个它的结构图 ArrayList作为一个集合工具,对于我而言它值得我们 ...
- In Java, how can I test if an Array contains a certain value?
public static final String[] VALUES = newString[]{"AB","BC","CD"," ...
- python基础语法15 面向对象2 继承,多态,继承json模块中JSONEncoder,并派生出新的功能
继承 1.什么是继承? 继承是一种新建类的方式,新建的类称之为子类或派生类,继承的父类称之为基类或超类. - 在Python中,一个子类可以继承多个父类.(面试可能会问) - 在其它语言中,一个子类只 ...
- 【转载】浅析从外部访问 Kubernetes 集群中应用的几种方式
一般情况下,Kubernetes 的 Cluster Network 是属于私有网络,只能在 Cluster Network 内部才能访问部署的应用.那么如何才能将 Kubernetes 集群中的应用 ...
- Python基础之while和for
实现ATM的输入密码重新输入的功能 while True: user_db = 'nick' pwd_db = '123' inp_user = input('username: ') inp_pwd ...