python 内存NoSQL数据库

来自于网络,经过修改,秉承Open Source精神,回馈网络!

#!/usr/bin/python
#-*- coding: UTF-8 -*-
#
# memdb.py
#   python memory db
#
# 2015-12
########################################################################
# The MIT License (MIT)
#    http://opensource.org/licenses/MIT
#
#  Copyright (c) 2015 copyright cheungmine
#
# Permission is hereby granted, free of charge, to any person obtaining
#  a copy of this software and associated documentation files (the
#  "Software"), to deal in the Software without restriction, including
#  without limitation the rights to use, copy, modify, merge, publish,
#  distribute, sublicense, and/or sell copies of the Software, and to
#  permit persons to whom the Software is furnished to do so, subject
#  to the following conditions:
#
# The above copyright notice and this permission notice shall be
#  included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
########################################################################

from multiprocessing import RLock

# sync for threading
class ObjectSync:
    def __init__(self, name):
        self.refcount = 0
        self.synclock = RLock()
        self.keyname  = name

    def Lock(self):
        self.synclock.acquire()
        self.refcount = self.refcount + 1

    def Unlock(self):
        self.refcount = self.refcount - 1
        self.synclock.release()

class ObjectSyncFactory:
    def __init__(self):
        self.globalLock = ObjectSync("")
        self.rowlocks = {}

    def __RemoveLock(self, sem, keyname):
        self.globalLock.Lock()
        self.rowlocks[keyname] = None
        self.globalLock.Unlock()

    def GetLock(self, tablename, key):
        keyname = tablename + "," + str(key)
        self.globalLock.Lock()

        l = None
        try:
            l = self.rowlocks[keyname]
            if l == None:
                self.rowlocks[keyname] = ObjectSync(keyname)
                l = self.rowlocks[keyname]
        except:
            self.rowlocks[keyname] = ObjectSync(keyname)
            l = self.rowlocks[keyname]

        self.globalLock.Unlock()

        return l

class PairGuard:
    def __init__(self, factory, sem):
        self.syncfactory = factory
        self.host = sem
        self.host.Lock()

    def __del__(self):
        self.host.Unlock()
        if self.host.refcount == 0 :
            self.syncfactory._ObjectSyncFactory__RemoveLock(self.host, self.host.keyname)

########################################
# Database table
class MemTable:
    def __init__(self):
        self.rows = {}
        self.tableLock = ObjectSync("")

    def GetRowCount(self):
        return len(self.rows)

    def DeleteAll(self):
        self.tableLock.Lock()
        self.rows = {}
        self.tableLock.Unlock()

    def __DeleteAll(self):
        self.rows = {}

    def GetAllValue(self):
        return self.rows

    # throw KeyError if key not found.
    def GetValue(self, key):
        return self.rows[key]

    # not exist: Add
    # exist: Update
    def AddValue(self, key, value):
        self.tableLock.Lock()
        self.rows[key] = value
        self.tableLock.Unlock()

    def __AddValue(self, key, value):
        self.rows[key] = value

    def DelValue(self, key):
        self.AddValue(key,None)

    def __DelValue(self, key):
        self._MemTable__AddValue(key, None)

########################################
# MemDB
class MemDB:
    def __init__(self):
        self.tables = {}
        self.syncFactory = ObjectSyncFactory()

    # is not thread safed
    def CreateTable(self, tablename):
        self.tables[tablename] = MemTable()

    # is not thread safed
    def DropTable(self, tablename):
        self.tables[tablename] = None

    def GetValue(self, tablename, key):
        mt = self.tables[tablename]
        PairGuard(self.syncFactory, self.syncFactory.GetLock(tablename, key))
        return mt.GetValue(key)

    def AddValue(self, tablename, key, value):
        mt = self.tables[tablename]
        PairGuard(self.syncFactory, self.syncFactory.GetLock(tablename, key))
        mt.AddValue(key, value)

    def DelValue(self, tablename, key):
        mt = self.tables[tablename]
        PairGuard(self.syncFactory, self.syncFactory.GetLock(tablename, key))
        mt.DelValue(key)

    def __GetValue(self, tablename, key):
        mt = self.tables[tablename]
        return mt.GetValue(key)

    def __AddValue(self, tablename, key, value):
        mt = self.tables[tablename]
        mt._MemTable__AddValue(key, value)

    def __DelValue(self, tablename, key):
        mt = self.tables[tablename]
        mt._MemTable__DelValue(key)

class Transaction:
    def __init__(self, conn):
        self.dbconn = conn
        self.logs = []

    def Commit(self):
        syncs = []
        tables = {}

        for p in self.logs:
            tables[p[0]] = True

        for name in tables:
            syncTable = self.dbconn.memdb.syncFactory.GetLock(name, 'table')
            syncs.append( (syncTable.keyname, syncTable) )

        syncs.sort()

        #lock
        guards = []
        for sync in syncs:
            guards.append(PairGuard(self.dbconn.memdb.syncFactory, sync[1]))

        #commit
        self.logs.reverse()
        while True:
            if len(self.logs) == 0:
                break
            p = self.logs.pop()
            self.dbconn.memdb._MemDB__AddValue(p[0], p[1], p[2])

        #unlock
        guards.reverse()
        while True:
            if len(guards) == 0:
                break
            guards.pop()

        self.dbconn._MemDBConnect__EndTransaction()

    def Rollback(self):
        self.dbconn._MemDBConnect__EndTransaction()

    def LogPoint(self, tablename, key, value):
        self.logs.append((tablename, key, value))

class MemDBConnect:
    def __init__(self, db):
        self.memdb = db
        self.bTransaction = False
        self.trans = None

    def BeginTransaction(self):
        self.bTransaction = True
        self.trans = Transaction(self)
        return self.trans

    def __EndTransaction(self):
        self.bTransaction = False
        self.trans = None

    def CommitTransaction(self):
        if self.bTransaction:
            self.bTransaction = False
            ts = self.trans
            self.trans = None
            if ts:
                ts.Commit()

    def RollbackTransaction(self):
        if self.bTransaction:
            self.bTransaction = False
            ts = self.trans
            self.trans = None
            if ts:
                ts.Rollback()

    # not thread safe
    def CreateTable(self, tablename):
        self.memdb.CreateTable(tablename)

    # not thread safe
    def DropTable(self, tablename):
        self.memdb.DropTable(tablename)

    # not thread safe
    def HasTable(self, tablename):
        if self.memdb.tables.get(tablename):
            return True
        else:
            return False

    def GetValue(self, tablename, key):
        if self.bTransaction:
            return self.memdb._MemDB__GetValue(tablename, key)
        else:
            return self.memdb.GetValue(tablename, key)

    def AddValue(self, tablename, key, value):
        if self.bTransaction:
            self.trans.LogPoint(tablename, key, value)
        else:
            self.memdb.AddValue(tablename, key, value)

    def DelValue(self, tablename, key):
        if self.bTransaction:
            self.trans.LogPoint(tablename, key, None)
        else:
            self.memdb.DelValue(tablename, key)

    def QueryTablesNothrow(self):
        tables = []
        try:
            self.BeginTransaction()

            for tablename,_ in self.memdb.tables.items():
                tables.append(tablename)

            self.CommitTransaction()
        except:
            tables = []
            self.RollbackTransaction()
        finally:
            return tables

    def QueryTableKeysNothrow(self, tablename):
        keys = []
        try:
            self.BeginTransaction()

            if self.HasTable(tablename):
                rows_dict = self.memdb.tables[tablename].rows
                for key, _ in rows_dict.items():
                    keys.append(key)

            self.CommitTransaction()
        except:
            keys = []
            self.RollbackTransaction()
        finally:
            return keys

    def CreateTableNothrow(self, tablename):
        try:
            self.BeginTransaction()

            if not self.HasTable(tablename):
                self.memdb.CreateTable(tablename)
                self.CommitTransaction()
        except:
            self.RollbackTransaction()
        finally:
            pass

    def DropTableNothrow(self, tablename):
        try:
            self.BeginTransaction()

            if self.HasTable(tablename):
                self.memdb.DropTable(tablename)
                self.CommitTransaction()
        except:
            self.RollbackTransaction()
        finally:
            pass

    def GetValueNothrow(self, tablename, key, defaultvalue):
        result = defaultvalue
        try:
            self.BeginTransaction()

            result = self.GetValue(tablename, key)

            self.CommitTransaction()
        except:
            self.RollbackTransaction()
        finally:
            return result

    def AddValueNothrow(self, tablename, key, value):
        result = False
        try:
            self.BeginTransaction()

            self.AddValue(tablename, key, value)

            self.CommitTransaction()

            result = True
        except:
            self.RollbackTransaction()
        finally:
            return result

    def DelValueNothrow(self, tablename, key):
        result = False
        try:
            self.BeginTransaction()

            self.DelValue(tablename, key)

            self.CommitTransaction()

            result = True
        except:
            self.RollbackTransaction()
        finally:
            return result

    def AppendValueListNothrow(self, tablename, key, value, non_repeated_value):
        try:
            self.BeginTransaction()

            if self.HasTable(tablename):
                try:
                    values = self.GetValue(tablename, key)

                    if non_repeated_value:
                        if value not in values:
                            values.append(value)
                            self.AddValue(tablename, key, values)
                    else:
                        values.append(value)
                        self.AddValue(tablename, key, values)
                except KeyError:
                    self.AddValue(tablename, key, [value])
                finally:
                    self.CommitTransaction()
        except:
            self.RollbackTransaction()
        finally:
            pass

    def AppendValueListMultiNothrow(self, tablenames, keys, values, non_repeated_values):
        try:
            self.BeginTransaction()

            for i in range(0, len(tablenames)):
                t, k, v, nrv = tablenames[i], keys[i], values[i], non_repeated_values[i]

                if self.HasTable(t):
                    try:
                        vals = self.GetValue(t, k)

                        if nrv:
                            if v not in vals:
                                vals.append(v)
                                self.AddValue(t, k, vals)
                        else:
                            vals.append(v)
                            self.AddValue(t, k, vals)
                    except KeyError:
                        self.AddValue(t, k, [v])

            self.CommitTransaction()
        except:
            self.RollbackTransaction()
        finally:
            pass

用法1:

可以直接嵌入到python中:

def test():
    db = MemDB()
    tname = "table1"
    db.CreateTable(tname)

    #for i in range(100000):
    #    db.AddValue(tname,i,"sdfsd")
    db.AddValue(tname,11,"sdfsd")
    print db.GetValue(tname, 11)

    db.AddValue(tname,11,"dddddd")
    print db.GetValue(tname,11)

    db.AddValue(tname,12,"dsfdsfd")
    print db.GetValue(tname,12)

    conn = MemDBConnect(db)
    t = conn.BeginTransaction()
    for i in range(100000):
        conn.AddValue(tname,i,"sdfsd")
    conn.AddValue(tname,12,"sfdas")
    conn.AddValue(tname,12,"ddddd")
    t.Commit()
    print db.GetValue(tname,12)

python 内存NoSQL数据库的更多相关文章

  1. Python操作nosql数据库之redis

    一.NoSQL的操作 NoSQL,泛指非关系型的数据库.随着互联网web2.0网站的兴起,传统的关系数据库在应付web2.0网站,特别是超大规模和高并发的SNS类型的web2.0纯动态网站已经显得力不 ...

  2. python操作nosql数据库之memcache

    一.memcache的安装 1.memcache简介 Memcached是一个高性能的分布式内存对象缓存系统,用于动态Web应用以减轻数据库负载.它通过在内存中缓存数据和对象减少读取数据库的次数,从而 ...

  3. 性能超越 Redis 的 NoSQL 数据库 SSDB

    idea's blog - 性能超越 Redis 的 NoSQL 数据库 SSDB 性能超越 Redis 的 NoSQL 数据库 SSDB C/C++语言编程, SSDB Views: 8091 | ...

  4. 孤荷凌寒自学python第五十天第一次接触NoSql数据库_Firebase

    孤荷凌寒自学python第五十天第一次接触NoSql数据库_Firebase (完整学习过程屏幕记录视频地址在文末) 之前对关系型数据库的学习告一段落,虽然能力所限没有能够完全完成理想中的所有数据库操 ...

  5. 用 Python 写一个 NoSQL 数据库Python

    NoSQL 这个词在近些年正变得随处可见. 但是到底 “NoSQL” 指的是什么? 它是如何并且为什么这么有用? 在本文, 我们将会通过纯 Python (我比较喜欢叫它, “轻结构化的伪代码”) 写 ...

  6. NoSQL数据库笔谈(转)

    NoSQL数据库笔谈 databases , appdir , node , paper颜开 , v0.2 , 2010.2 序 思想篇 CAP 最终一致性 变体 BASE 其他 I/O的五分钟法则 ...

  7. 15个nosql数据库

    1.MongoDB 介绍 MongoDB是一个基于分布式文件存储的数据库.由C++语言编写.主要解决的是海量数据的访问效率问题,为WEB应用提供可扩展的高性能数据存储解决方案.当数据量达到50GB以上 ...

  8. NoSQL 数据库产品学习总结(一)

    NoSQL 数据库产品学习总结(一) 本篇文章共分为四个章节,会陆续整理下 Memcached.Redis.tair.mongodb.hbase.SequoiaDB. Cassandra的相关知识. ...

  9. NoSQL 数据库的使用场景

    摘要:对比传统关系型数据库,NoSQL有着更为复杂的分类——键值.面向文档.列存储.图数据库.这里就带你一览NoSQL各种类型的适用场景及一些知名公司的方案选择. 在过去几年,关系型数据库一直是数据持 ...

随机推荐

  1. SpringBoot 使用MultipartFile上传文件相关问题解决方案

    1.当上传时未配置上传内容大小,会报错[org.apache.tomcat.util.http.fileupload.FileUploadBase$SizeLimitExceededException ...

  2. Linux 下 HTTP连接超时

    将项目部署到现场环境,HTTP请求莫名奇妙的连接超时,通过抓包定位了问题,是请求的IP被禁止掉.其中用到了抓包,将记录记录于此. tcpdump host 120.197.89.51 -i any - ...

  3. chrome 如何卸载干净

    安装位置C:\Users\你电脑的用户名\AppData\Local\Google,删除整个文件夹,用CCleaner扫描注册表删除无用注册表项,重启安装即可.

  4. 解决com.fasterxml.jackson.databind.JsonMappingException: No suitable

    原因:直接翻译,json格式,不匹配. 这原因坑爹啊,因为json格式不正确算一种原因. 还有一种就是接收的bean没有getter,setter方法. 最坑的一种就是数据无法被反序列化,list,m ...

  5. spark学习笔记01

    spark学习笔记01 1.课程目标 1.熟悉spark相关概念 2.搭建一个spark集群 3.编写简单spark应用程序 2.spark概述 spark是什么 是基于内存的分布式计算引擎,计算速度 ...

  6. ECC公钥格式详解

    本文首先介绍公钥格式相关的若干概念/技术,随后以示例的方式剖析DER格式的ECC公钥,最后介绍如何使用Java生成.解析和使用ECC公钥. ASN.1 Abstract Syntax Notation ...

  7. Qt与FFmpeg联合开发指南(二)——解码(2):封装和界面设计

    与解码相关的主要代码在上一篇博客中已经做了介绍,本篇我们会先讨论一下如何控制解码速度再提供一个我个人的封装思路.最后回归到界面设计环节重点看一下如何保证播放器界面在缩放和拖动的过程中保证视频画面的宽高 ...

  8. git使用之错误分析及解决(持续更新)

    错误一: 使用 $ git push -u origin master 出现如下错误: error: src refspec master does not match any. error: fai ...

  9. Bootstrap3 代码-变量

    通过 <var> 标签标记变量. y = mx + b <var>y</var> = <var>m</var><var>x< ...

  10. Dynamics CRM2016 Web API之获取查找字段的text及选项集的text

    本篇再来介绍个web api的功能,关于lookup的text这里只是略带,因为有expand,现有的web api就能实现,主要提的是选项集的text,我们通过基本的查询api查出来的字段值只带有v ...