improve docker-compose sample

config_example.json

@@ -1,11 +1,13 @@
 {
-  "taskdb": "couchdb+taskdb://couchdb:5984",
-  "projectdb": "couchdb+projectdb://couchdb:5984",
-  "resultdb": "couchdb+resultdb://couchdb:5984",
+  "taskdb": "couchdb+taskdb://user:password@couchdb:5984",
+  "projectdb": "couchdb+projectdb://user:password@couchdb:5984",
+  "resultdb": "couchdb+resultdb://user:password@couchdb:5984",
   "message_queue": "amqp://rabbitmq:5672/%2F",
   "webui": {
     "username": "username",
     "password": "password",
-    "need-auth": true
+    "need-auth": true,
+    "scheduler-rpc": "http://scheduler:23333",
+    "fetcher-rpc": "http://fetcher:24444"
   }
-}
+}

docker-compose.yaml

@@ -16,17 +16,13 @@ services:
   couchdb:
     image: couchdb:latest
     container_name: couchdb
+    environment:
+      - COUCHDB_USER=user
+      - COUCHDB_PASSWORD=password
     networks:
       - pyspider
     ports:
       - "5984:5984"
-    environment:
-      - COUCHDB_NAME=couchdb
-      - COUCHDB_PORT_5984_TCP_ADDR=couchdb
-      - COUCHDB_PORT_5984_TCP_PORT=5984
-      - COUCHDB_USER=user
-      - COUCHDB_PASSWORD=password
-      #- COUCHDB_HTTPS=true # enable if running couchdb over https
   # OR we can replace couchdb with mysql
   #mysql:
   #  image: mysql:latest
@@ -43,14 +39,8 @@ services:
     networks:
       - pyspider
     volumes:
-      - /path/to/dir/config.json:/opt/pyspider/config.json
+      - ./config_example.json:/opt/pyspider/config.json
     command: -c config.json phantomjs
-    environment:
-      - COUCHDB_NAME=couchdb
-      - COUCHDB_PORT_5984_TCP_ADDR=couchdb
-      - COUCHDB_PORT_5984_TCP_PORT=5984
-      - COUCHDB_USER=user
-      - COUCHDB_PASSWORD=password
     depends_on:
       - couchdb
       - rabbitmq
@@ -61,14 +51,8 @@ services:
     networks:
       - pyspider
     volumes:
-      - /path/to/dir/config.json:/opt/pyspider/config.json
+      - ./config_example.json:/opt/pyspider/config.json
     command: -c config.json result_worker
-    environment:
-      - COUCHDB_NAME=couchdb
-      - COUCHDB_PORT_5984_TCP_ADDR=couchdb
-      - COUCHDB_PORT_5984_TCP_PORT=5984
-      - COUCHDB_USER=user
-      - COUCHDB_PASSWORD=password
     depends_on:
       - couchdb
       - rabbitmq
@@ -79,14 +63,8 @@ services:
     networks:
       - pyspider
     volumes:
-      - /path/to/dir/config.json:/opt/pyspider/config.json
+      - ./config_example.json:/opt/pyspider/config.json
     command: -c config.json processor
-    environment:
-      - COUCHDB_NAME=couchdb
-      - COUCHDB_PORT_5984_TCP_ADDR=couchdb
-      - COUCHDB_PORT_5984_TCP_PORT=5984
-      - COUCHDB_USER=user
-      - COUCHDB_PASSWORD=password
     depends_on:
       - couchdb
       - rabbitmq
@@ -97,14 +75,8 @@ services:
     networks:
       - pyspider
     volumes:
-      - /path/to/dir/config.json:/opt/pyspider/config.json
+      - ./config_example.json:/opt/pyspider/config.json
     command : -c config.json fetcher
-    environment:
-      - COUCHDB_NAME=couchdb
-      - COUCHDB_PORT_5984_TCP_ADDR=couchdb
-      - COUCHDB_PORT_5984_TCP_PORT=5984
-      - COUCHDB_USER=user
-      - COUCHDB_PASSWORD=password
     depends_on:
       - couchdb
       - rabbitmq
@@ -115,14 +87,8 @@ services:
     networks:
       - pyspider
     volumes:
-      - /path/to/dir/config.json:/opt/pyspider/config.json
+      - ./config_example.json:/opt/pyspider/config.json
     command: -c config.json scheduler
-    environment:
-      - COUCHDB_NAME=couchdb
-      - COUCHDB_PORT_5984_TCP_ADDR=couchdb
-      - COUCHDB_PORT_5984_TCP_PORT=5984
-      - COUCHDB_USER=user
-      - COUCHDB_PASSWORD=password
     depends_on:
       - couchdb
       - rabbitmq
@@ -135,14 +101,7 @@ services:
     networks:
       - pyspider
     volumes:
-      - /path/to/dir/config.json:/opt/pyspider/config.json
-    environment:
-      - SCHEDULER_PORT_23333_TCP_ADDR=scheduler
-      - COUCHDB_NAME=couchdb
-      - COUCHDB_PORT_5984_TCP_ADDR=couchdb
-      - COUCHDB_PORT_5984_TCP_PORT=5984
-      - COUCHDB_USER=user
-      - COUCHDB_PASSWORD=password
+      - ./config_example.json:/opt/pyspider/config.json
     command: -c config.json webui
     depends_on:
       - couchdb
@@ -154,4 +113,4 @@ networks:
     external:
       name: pyspider
   default:
-    driver: bridge
+    driver: bridge

pyspider/__init__.py

@@ -5,4 +5,4 @@
 #         http://binux.me
 # Created on 2014-11-17 19:17:12
 
-__version__ = '0.3.10'
+__version__ = '0.4.0'

pyspider/database/__init__.py

@@ -213,18 +213,9 @@ def _connect_couchdb(parsed, dbtype, url):
         url = "http://" + parsed.netloc + "/"
     params = {}
 
-    username = None
-    password = None
-    if '@' in parsed.netloc:
-        # netloc looks like: 'user:pass@couchdb:999'
-        url = parsed.netloc[parsed.netloc.find("@")+1:]
-        # extract the username and password
-        username = parsed.netloc[:parsed.netloc.find(":")]
-        password = parsed.netloc[parsed.netloc.find(":")+1:parsed.netloc.find("@")]
-
     # default to env, then url, then hard coded
-    params['username'] = os.environ.get('COUCHDB_USER') or username or 'user'
-    params['password'] = os.environ.get('COUCHDB_PASSWORD') or password or 'password'
+    params['username'] = os.environ.get('COUCHDB_USER') or parsed.username or 'user'
+    params['password'] = os.environ.get('COUCHDB_PASSWORD') or parsed.password or 'password'
 
     # create necessary DBs + the admin user
     res = requests.put(url + "_users")
@@ -254,4 +245,4 @@ def _connect_couchdb(parsed, dbtype, url):
         from .couchdb.resultdb import ResultDB
         return ResultDB(url, **params)
     else:
-        raise LookupError
+        raise LookupError