mongodbの最近のブログ記事

下記がコードサンプル、create_indexは忘れずに。

#!/usr/bin/env python

import binascii
import struct
import datetime
import hashlib
rom pymongo import Connection
from time import sleep

reconnect_times = 10
reconnect_counter = 0
dbsetting = {'dbhost':'localhost','dbname':'shorturldb'}

def con(**kwargs):

    global reconnect_counter
    global reconnect_times

    if 'host' not in kwargs:
        kwargs['host'] = dbsetting['dbhost']
    dbname = dbsetting['dbname']

    try:
        _con = Connection(**kwargs)
        return eval('_con.'+dbname)

    except Exception, e:

        sleep(0.1)

        print('Fail to excute %s [%s]' % ( __name__, e))
        if reconnect_counter == reconnect_times:
            reconnect_counter = reconnect_counter + 1
            con(**kwargs)


def save_hash(url, length=5):
    base32 = [
            'a', 'b', 'c', 'd', 'e',
            'f', 'g', 'h', 'i', 'j',
            'k', 'l', 'm', 'n', 'o',
            'p', 'q', 'r', 's', 't',
            'u', 'v', 'w', 'x', 'y','z',
            'A', 'B', 'C', 'D', 'E',
            'F', 'G', 'H', 'I', 'J',
            'K', 'L', 'M', 'N', 'O',
            'P', 'Q', 'R', 'S', 'T',
            'U', 'V', 'W', 'X', 'Y','Z',
            '0', '1', '2', '3', '4',
            '5', '6', '7', '8', '9'
            ]
    hex = hashlib.md5(url).hexdigest()
    hexLen = len(hex)
    subHexLen = hexLen/8
    output = []
    mydb = con()
    for i in range(0, subHexLen):
        subHex = hex[i*8:(i*8)+8]
        num = 0x3FFFFFFF & int('0x'+subHex, 0)
        hash = '';
        for j in range(0,length):
            val = 0x0000003D & num
            hash = hash + base32[val]
            num = num >> length
        r = mydb.shorturl.find_one({'i':{'$in':[hash,url]}})
        if not r:
            insert = {'i':[hash,url],'p':datetime.datetime.utcnow()}
            mydb.shorturl.insert(insert)
            return hash

        elif r and r['i'][1] == url:
            return r['i'][0]
    return None

def get_hash(url):
    mydb = con()
    r = mydb.shorturl.find_one({'i':{'$in':[url]}})
    if r:
        return r['i'][0]
    return None

from pymongo import Connection
from time import sleep

reconnect_times = 10
reconnect_counter = 0

def con(**kwargs):

    global reconnect_counter
    global reconnect_times

    if 'host' not in kwargs:
        kwargs['host'] = settings.application['dbhost']
    dbname = settings.application['dbname']

    try:
        _con = Connection(**kwargs)
        return eval('_con.'+dbname)

    except Exception, e:

        sleep(0.1)

        print('Fail to excute %s [%s]' % ( __name__, e))
        if reconnect_counter == reconnect_times:
            reconnect_counter = reconnect_counter + 1
            con(**kwargs)

==
mongodb 1.5.* は若干バギーで、コネクトエラーがたまにでる。（でるらしいので、pymongoのコネクトラッパーらしきものもあるが）自前で、対応

mongosが安定してきたので一安心。

==
ディレクトリ作る（ec2の場合とりあえず、/mnt以下へ)
==

$ sudo mkdir /mnt/data/shards
$ sudo mkdir /mnt/data/shards/0 /mnt/data/shards/1 /mnt/data/shards/2
$ sudo mkdir /mnt/data/shards/config
$ sudo chown mongodb:mongodb -R /mnt/data/shards

==
mongosの設定
==

$ sudo -u mongodb -H mongo
> use admin
> db.runCommand( {addshard : "localhost:27020", allowLocal : true} );
> db.runCommand( {addshard : "localhost:27021", allowLocal : true} );
> db.runCommand( {addshard : "localhost:27022", allowLocal : true} );

# Listing shards
> db.runCommand( { listshards : 1 } );

#Enabling Sharding on a Database
> db.runCommand( { enablesharding : "dbname" } );

#Sharding a Collection
> db.runCommand( { shardcollection : "dbname.collection_name" , key : { "_id" : 1 } , unique : true } );

==
簡易スタートスクリプト
==

#!/bin/bash

sudo -u mongodb -H mongod \
--configsvr \
--dbpath /mnt/data/shards/config \
--fork \
--port 27019 \
--logpath /var/log/mongodb/mongodb.config.log  &

sleep 2

sudo mongos \
-vvv \
--port 27017 \
--fork \
--configdb localhost:27019 \
--logpath /var/log/mongodb/mongos.log  &

sleep 2

sudo -u mongodb -H mongod \
--shardsvr \
--port 27020 \
--fork \
--dbpath /mnt/data/shards/0 \
--logpath /var/log/mongodb/mongodb.shard.log &

sudo -u mongodb -H mongod \
--shardsvr \
--port 27021 \
--fork \
--dbpath /mnt/data/shards/1 \
--logpath /var/log/mongodb/mongodb.shard.log &

sudo -u mongodb -H mongod \
--shardsvr \
--port 27022 \
--fork \
--dbpath /mnt/data/shards/2 \
--logpath /var/log/mongodb/mongodb.shard.log  &

print '-------------------------------------'
ps -o pid,command ax | grep mongos | grep -v grep |cut -d' ' -f1
print '-------------------------------------'

ps -o pid,command ax | grep mongos | grep -v grep |cut -d' ' -f1 > /tmp/mongo.pid

2 servers results

Web Sever: Nginx 0.7* + Tornado 0.2
                   using nginx 5 load balancers
                  worker processes 5
                  worker connections 1024

DB Server: mongodb 1.5.1 shards

====
$ ab -n 10000 -c 25 http://****.***/
This is ApacheBench, Version 2.3 <$Revision: 655654 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/

Benchmarking ****.*** (be patient)
Completed 1000 requests
Completed 2000 requests
Completed 3000 requests
Completed 4000 requests
Completed 5000 requests
Completed 6000 requests
Completed 7000 requests
Completed 8000 requests
Completed 9000 requests
Completed 10000 requests
Finished 10000 requests


Server Software:        TornadoServer/0.1
Server Hostname:        ****.***
Server Port:            80

Document Path:          /
Document Length:        4259 bytes

Concurrency Level:      25
Time taken for tests:   139.086 seconds
Complete requests:      10000
Failed requests:        1469
   (Connect: 0, Receive: 0, Length: 1469, Exceptions: 0)
Write errors:           0
Non-2xx responses:      1469
Total transferred:      38561669 bytes
HTML transferred:       36470146 bytes
Requests per second:    71.90 [#/sec] (mean)
Time per request:       347.716 [ms] (mean)
Time per request:       13.909 [ms] (mean, across all concurrent requests)
Transfer rate:          270.75 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:      106  108  42.5    107    3106
Processing:   108  239 221.4    200    3539
Waiting:      108  239 221.3    200    3539
Total:        214  347 225.2    307    3646

Percentage of the requests served within a certain time (ms)
  50%    307
  66%    356
  75%    392
  80%    419
  90%    482
  95%    530
  98%    602
  99%    664
 100%   3646 (longest request)

==
mongodbが2回ほど、open fileを振り切って、connectionエラー、 ulimitをマックスまであげるが同じ。
なので、monitを使って、can not open fileになったら、サーバーリスタートで対応。

xcode  と  mac port をインストール

そして

$ sudo port install boost pcre
$ sudo port install spidermonkey
$ sudo port mongo

# ついでにpymongoも
$ sudo easy_install pymongo

参照: mongodb
http://www.mongodb.org/display/DOCS/Building+for+OS+X?showComments=true&showCommentArea=true

tornadoも同じ、(xcodeを入れてコンパイル環境を用意する)

で、
$ sudo easy_install setuptools pycurl==7.16.2.1 simplejson
$ git cloan git://github.com/facebook/tornado.git
$ cd tornado
$ python setup.py build
$ sudo python setup.py install

mongodbのデーモン化シェルのサンプルです。

==
#! /bin/sh

### BEGIN INIT INFO
# Provides:          mongo
# Required-Start:    $all
# Required-Stop:     $all
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: starts the mongodb server
# Description:       starts mongo using start-stop-daemon
### END INIT INFO

PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
DAEMON=/usr/local/mongo/bin/mongod
NAME=mongo
DESC='mongodb server'
# ここは/etc/defalut/mongoでincludeしてもよいかも
DAEMON_OPTS='--dbpath /var/lib/mongo'

test -x $DAEMON || exit 0

set -e

case "$1" in
  start)
        echo -n "Starting $DESC: "
        start-stop-daemon --start --quiet --pidfile /var/run/$NAME.pid --make-pidfile --background \
                --exec $DAEMON -- $DAEMON_OPTS
        echo "$NAME."
        ;;
  stop)
        echo -n "Stopping $DESC: "
        start-stop-daemon --stop --quiet --pidfile /var/run/$NAME.pid \
                --exec $DAEMON
        echo "$NAME."
        ;;
  restart|force-reload)
        echo -n "Restarting $DESC: "
        start-stop-daemon --stop --quiet --pidfile \
                /var/run/$NAME.pid --exec $DAEMON
        sleep 1
        start-stop-daemon --start --quiet --pidfile \
                /var/run/$NAME.pid --exec $DAEMON -- $DAEMON_OPTS
        echo "$NAME."
        ;;
  reload)
      echo -n "Reloading $DESC configuration: "
      start-stop-daemon --stop --signal HUP --quiet --pidfile /var/run/$NAME.pid \
          --exec $DAEMON
      echo "$NAME."
      ;;
  *)
        N=/etc/init.d/$NAME
        echo "Usage: $N {start|stop|restart|reload|force-reload}" >&2
        exit 1
        ;;
esac

exit 0

mongodbを検証するためにインストール

1. 必要なパッケージのインストール
sudo apt-get install tcsh git-core scons g++
sudo apt-get install libpcre++-dev libboost1.37-dev libreadline-dev libmozjs-dev xulrunner-dev

2. ダウンロードしてインストール
cd /usr/local
git clone git://github.com/mongodb/mongo.git
cd mongo
sudo scons all
sudo scons --prefix=/opt/mongo install

3. 実験用に任意のデータディレクトを作る
mkdir data

4. デーモンを起動

sudo /opt/mongo/bin/mongod --dbpath /path/to/db &

5. pymongoをインストール
sudo easy_install pymongo

6. 接続テスト
http://pypi.python.org/pypi/pymongo/

この辺りも参照：
http://www.mongodb.org/display/DOCS/Building+for+Linux

かなり手軽に