WIP - feedback requested - multipool support

.travis.yml

@@ -1,6 +1,7 @@
 language: erlang
 
 otp_release:
+  - 18.1
   - 17.0
   - R16B03-1
   - R16B03

common_test/pgsql_create.sql

@@ -1,8 +1,20 @@
-CREATE USER itest;
-CREATE DATABASE itest OWNER itest;
-GRANT ALL PRIVILEGES ON DATABASE itest TO itest;
+CREATE USER itest_sqerl1 WITH ENCRYPTED PASSWORD 'itest_sqerl1';
+CREATE USER itest_sqerl2 WITH ENCRYPTED PASSWORD 'itest_sqerl2';
+
+
+CREATE DATABASE itest_sqerl1 OWNER itest_sqerl1;
+CREATE DATABASE itest_sqerl2 OWNER itest_sqerl2;
+
+\c itest_sqerl2;
+GRANT ALL PRIVILEGES ON DATABASE itest_sqerl2 TO itest_sqerl2;
+CREATE TABLE only_in_itest_sqerl2_db ( name VARCHAR(128));
+GRANT ALL PRIVILEGES ON TABLE only_in_itest_sqerl2_db TO itest_sqerl2;
+
+\c itest_sqerl1;
+GRANT ALL PRIVILEGES ON DATABASE itest_sqerl1 TO itest_sqerl1;
+CREATE TABLE only_in_itest_sqerl1_db ( name VARCHAR(128));
+GRANT ALL PRIVILEGES ON TABLE only_in_itest_sqerl1_db TO itest_sqerl1;
 
-\c itest;
 CREATE SEQUENCE users_id_sequence;
 /* Create test tables */
 CREATE TABLE users (
@@ -15,8 +27,8 @@ CREATE TABLE users (
        created timestamp
 );
 
-GRANT ALL PRIVILEGES ON TABLE users TO itest;
-GRANT ALL PRIVILEGES ON SEQUENCE users_id_sequence TO itest;
+GRANT ALL PRIVILEGES ON TABLE users TO itest_sqerl1;
+GRANT ALL PRIVILEGES ON SEQUENCE users_id_sequence TO itest_sqerl1;
 
 CREATE TABLE nodes (
        id char(32) PRIMARY KEY,
@@ -30,7 +42,7 @@ CREATE TABLE nodes (
        updated_at timestamp NOT NULL
 );
 
-GRANT ALL PRIVILEGES ON TABLE nodes TO itest;
+GRANT ALL PRIVILEGES ON TABLE nodes TO itest_sqerl1;
 
 CREATE OR REPLACE FUNCTION insert_users(varchar[],
     varchar[],
@@ -57,7 +69,7 @@ CREATE TABLE uuids (
        id uuid UNIQUE NOT NULL
 );
 
-GRANT ALL PRIVILEGES ON TABLE uuids TO itest;
+GRANT ALL PRIVILEGES ON TABLE uuids TO itest_sqerl1;
 
 CREATE OR REPLACE FUNCTION insert_ids(uuid[])
 RETURNS VOID AS
@@ -73,4 +85,3 @@ END;
 $$
 LANGUAGE plpgsql;
 
-

common_test/pgsql_test_buddy.erl

@@ -6,8 +6,15 @@
 -compile([export_all]).
 
 -define(USER, string:strip(os:cmd("echo $USER"), right, $\n)).
+% NOTE - do not merge.  Doing this because kitchen isn't syncing up my local changes
+% and spending my time debugging secondary issues (or completely cycling the kitchen boxes
+% for each test) isn't something I'm inclined to do at the moment..
 -define(DB_PIPE_CMD, "psql -q -d postgres -h localhost -p 5432 -U " ++ ?USER ++ " -f - < ~s").
 -define(DB_CMD, "psql -h localhost -p 5432 -U " ++ ?USER ++ " -d postgres -w -c '~s'").
+%-define(DB_PIPE_CMD, "sudo -u postgres psql -q -d postgres -f - < ~s").
+%-define(DB_CMD, "psql -d postgres -w -c '~s'").
+% END NOTE
+
 
 -define(POOL_NAME, sqerl).
 -define(POOLER_TIMEOUT, 500).
@@ -17,40 +24,29 @@
 
 clean() ->
     [ os:cmd(io_lib:format(?DB_CMD, [Cmd])) || Cmd <- [
-        "drop database if exists itest",
-        "drop user if exists itest"
+        "drop database if exists itest_sqerl1",
+        "drop database if exists itest_sqerl2",
+        "drop user if exists itest_sqerl1",
+        "drop user if exists itest_sqerl2"
     ]].
 
-create(Config) ->
+config_file(Config, File) ->
     Dir = lists:reverse(filename:split(?config(data_dir, Config))),
     {_, Good} = lists:split(1, Dir),
-    File = filename:join(lists:reverse(Good) ++ ["pgsql_create.sql"]),
-    ct:pal("File: ~s", [File]),
-    os:cmd(io_lib:format(?DB_PIPE_CMD, [File])).
+    filename:join(lists:reverse(Good) ++ [File]).
+
+create(Config) ->
+    File1 = config_file(Config, "pgsql_create.sql"),
+    ct:pal("Results of loading ~s", [File1]),
+    ct:pal( os:cmd(io_lib:format(?DB_PIPE_CMD, [File1]))).
 
 setup_env() ->
     application:stop(sasl),
-    Info = db_config(),
-    ok = application:set_env(sqerl, db_driver_mod, sqerl_pgsql_client),
-    ok = application:set_env(sqerl, db_host, ?GET_ARG(host, Info)),
-    ok = application:set_env(sqerl, db_port, ?GET_ARG(port, Info)),
-    ok = application:set_env(sqerl, db_user, "itest"),
-    ok = application:set_env(sqerl, db_pass, "itest"),
-    ok = application:set_env(sqerl, db_name, ?GET_ARG(db, Info)),
-    ok = application:set_env(sqerl, idle_check, 10000),
-    ok = application:set_env(sqerl, pooler_timeout, ?POOLER_TIMEOUT),
-    %% we could also call it like this:
-    %% {prepared_statements, statements()},
-    %% {prepared_statements, "itest/statements_pgsql.conf"},
-    ok = application:set_env(sqerl, prepared_statements, {obj_user, '#statements', []}),
-    ColumnTransforms = [{<<"created">>,
-                         fun sqerl_transformers:convert_YMDHMS_tuple_to_datetime/1}],
-    ok = application:set_env(sqerl, column_transforms, ColumnTransforms),
-    PoolConfig = [{name, ?POOL_NAME},
-                  {max_count, ?MAX_POOL_COUNT},
-                  {init_count, 1},
-                  {start_mfa, {sqerl_client, start_link, []}}],
-    ok = application:set_env(pooler, pools, [PoolConfig]),
+    % By default we're going to set up multi-pool - this gives defacto verification
+    % that nothing gets broken in existing code (or tests) when the pool name is not
+    % specified.
+    %
+    setup_config(),
     Apps = [crypto, asn1, public_key, ssl, epgsql, pooler],
     [ application:start(A) || A <- Apps ],
 
@@ -59,20 +55,69 @@ setup_env() ->
     ?assert(lists:member(Status, [ok, {error, {already_started, sqerl}}])),
     ok.
 
+set_env_for(Key, EnvEntries) ->
+    [ application:set_env(Key, Entry, Value) || {Entry, Value} <- EnvEntries ].
+
 teardown_env() ->
     Apps = lists:reverse([crypto, asn1, public_key, ssl, epgsql, pooler, sqerl]),
     [application:stop(A) || A <- Apps].
 
-db_config() ->
-    [
-        {host, "localhost"},
-        {port, 5432},
-        {db, "itest"}
-    ].
+
+setup_config() ->
+
+    Pool1Extras = [{column_transforms,  [{<<"created">>, fun sqerl_transformers:convert_YMDHMS_tuple_to_datetime/1}]},
+                       {prepared_statements,  {obj_user, '#statements', []}} ],
+    Pool2Extras = [{column_transforms, []}, {prepared_statements, none}],
+    EnvCfg = config([
+                     {other, "itest_sqerl2", Pool2Extras},
+                     {sqerl, "itest_sqerl1", Pool1Extras}
+                    ]),
+    set_env_for(sqerl, ?config(sqerl, EnvCfg)),
+    set_env_for(pooler, ?config(pooler, EnvCfg)),
+    ct:pal("Environment configuration: ~p", [[{sqerl, application:get_all_env(sqerl)},
+                                              {pooler, application:get_all_env(pooler)}]]).
+config(DBInfo) ->
+    [{sqerl, [
+              {ip_mode, [ ipv4 ] },
+              {db_driver_mod, sqerl_pgsql_client},
+              {pooler_timeout, ?POOLER_TIMEOUT},
+              {databases, [ sqerl_db_config(Id, Name, Extras) || {Id, Name, Extras} <- DBInfo ]}
+             ]},
+     {pooler, [
+               {pools, [ pool_config(Id) || {Id, _, _} <- DBInfo ]}
+               %{metrics_module, folsom_metrics}
+              ]
+    }].
+
+sqerl_db_config(Id, TheName, Extras) ->
+    {Id,  lists:flatten([{db_host, "127.0.0.1"},
+                         {db_port, 5432},
+                         {db_user, TheName},
+                         {db_pass, TheName},
+                         {db_name, TheName},
+                         {db_timeout, 5000},
+                         {idle_check, 1000},
+                         {id, Id}, % debugging
+                         Extras ])
+    }.
+
+pool_config(Id) ->
+   [{name, Id},
+    {max_count, ?MAX_POOL_COUNT},
+    {init_count, 1},
+    {start_mfa, {sqerl_client, start_link, [{pool, Id}]}},
+    {queue_max, 5}].
 
 kill_pool() -> kill_pool(?MAX_POOL_COUNT).
 kill_pool(1) ->
     pooler:take_member(?POOL_NAME, ?POOLER_TIMEOUT);
 kill_pool(X) ->
     pooler:take_member(?POOL_NAME, ?POOLER_TIMEOUT),
     kill_pool(X - 1).
+
+get_user() ->
+    case os:getenv("PG_USER") of
+        false -> os:getenv("USER");
+        User -> User
+    end.
+

common_test/sqerl_integration_SUITE.erl

@@ -14,7 +14,7 @@
 
 all() -> [pool_overflow, insert_data, insert_returning, select_data, select_data_as_record,
             select_first_number_zero, delete_data, update_datablob, select_boolean, update_created,
-            select_simple, adhoc_select, adhoc_insert, insert_select_gzip_data, array_test,
+            select_simple, select_simple_multipool_, adhoc_select, adhoc_insert, insert_select_gzip_data, array_test,
             select_timeout, execute_timeout].
 
 init_per_testcase(_, Config) ->
@@ -133,12 +133,18 @@ select_simple(Config) ->
     {ok, Count} = sqerl:execute(SqlCount),
     [[{<<"num_users">>, 5}]] = Count,
 
+
+
     %% select_simple_with_parameters() ->
     Sql = <<"select id from users where last_name = $1">>,
     {ok, Rows} = sqerl:execute(Sql, ["Smith"]),
     ExpectedRows = [[{<<"id">>,1}]],
     ?assertEqual(ExpectedRows, Rows).
 
+select_simple_multipool_(_Config) ->
+    [ ?_assertMatch(ok, 0), sqerl:execute_with(sqerl:make_context(other), <<"SELECT COUNT(*) FROM only_in_itest_sqerl2_db">>),
+      ?_assertMatch(ok, 0), sqerl:execute_with(sqerl:make_context(sqerl), <<"SELECT COUNT(*) FROM only_in_itest_sqerl1_db">>)].
+
 adhoc_select(Config) ->
     insert_data(Config),
     insert_returning(Config),
@@ -361,6 +367,15 @@ adhoc_select(Config) ->
                        ],
         ?assertEqual(ExpectedRows, Rows)
     end(),
+    %% adhoc_select_limit
+    fun() ->
+        {ok, Rows} = sqerl:adhoc_select_with(sqerl:make_context(sqerl), [<<"id">>], <<"users">>, all, [{order_by, [<<"id">>]}, {limit, 2}]),
+        ExpectedRows = [
+                        [{<<"id">>, 1}],
+                        [{<<"id">>, 2}]
+                       ],
+        ?assertEqual(ExpectedRows, Rows)
+    end(),
 
     %% adhoc_select_offset
     fun() ->
@@ -380,7 +395,7 @@ adhoc_insert_delete_test(Table, Columns, Data, BatchSize) ->
     Values = [Value || [Value|_] <- Data],
     Where = {Field, in, Values},
     {ok, Rows} = sqerl:adhoc_select(Columns, Table, Where),
-    {ReturnedColumns, ReturnedData} = sqerl:extract_insert_data(Rows),
+    {ReturnedColumns, ReturnedData} = sqerl_core:extract_insert_data(Rows),
     %% clean up before asserts
     {ok, DeleteCount} = sqerl:adhoc_delete(Table, Where),
     %% now verify...

include/sqerl.hrl

@@ -31,8 +31,14 @@
                          {ok, integer(), sqerl_rows()} |
                          {error, atom() | tuple()}.
 
+-record(sqerl_ctx, { pool :: atom() }).
+-type sqerl_ctx() :: #sqerl_ctx{}.
+
 -ifdef(namespaced_types).
 -type sqerl_dict() :: dict:dict().
 -else.
 -type sqerl_dict() :: dict().
 -endif.
+
+-define(SQERL_DEFAULT_BATCH_SIZE, 100).
+-define(SQERL_ADHOC_INSERT_STMT_ATOM, '__adhoc_insert').