Overloading postgres on build

classic Classic list List threaded Threaded
9 messages Options
Reply | Threaded
Open this post in threaded view
|

Overloading postgres on build

JD Wong
Hi all,

It's been a while since the last help request, and so it's time to break that trend.  

I've recently expanded my build to include the genomic and protein fasta sources for all the species in wormbase along the lines of 48 fasta based sources .  

Loading them, I get this new stack trace.

It seems that the mine might be opening a new database connection for each source.  Can anybody comment on the root cause & how to fix it?

Thanks!

BUILD FAILED
/nfs/wormbase/wormmine/website-intermine-dev/acedb-dev/intermine/imbuild/integrate.xml:54: The following error occurred while executing this line:
/nfs/wormbase/wormmine/website-intermine-dev/acedb-dev/intermine/bio/sources/fasta/build.xml:46: org.intermine.objectstore.ObjectStoreException: Failed to instantiate IntegrationWriter class: org.intermine.dataloader.IntegrationWriterDataTrackingImpl
        at org.intermine.dataloader.IntegrationWriterFactory.getIntegrationWriter(IntegrationWriterFactory.java:75)
        at org.intermine.task.DirectDataLoaderTask.getIntegrationWriter(DirectDataLoaderTask.java:85)
        at org.intermine.task.DirectDataLoaderTask.execute(DirectDataLoaderTask.java:129)
        at org.intermine.bio.dataconversion.FastaLoaderTask.execute(FastaLoaderTask.java:200)
        at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java:291)
        at sun.reflect.GeneratedMethodAccessor360.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:597)
        at org.apache.tools.ant.dispatch.DispatchUtils.execute(DispatchUtils.java:106)
        at org.apache.tools.ant.Task.perform(Task.java:348)
        at org.apache.tools.ant.Target.execute(Target.java:390)
        at org.apache.tools.ant.Target.performTasks(Target.java:411)
        at org.apache.tools.ant.Project.executeSortedTargets(Project.java:1360)
        at org.apache.tools.ant.helper.SingleCheckExecutor.executeTargets(SingleCheckExecutor.java:38)
        at org.apache.tools.ant.Project.executeTargets(Project.java:1212)
        at org.apache.tools.ant.taskdefs.Ant.execute(Ant.java:441)
        at org.intermine.task.Integrate.performAction(Integrate.java:223)
        at org.intermine.task.Integrate.performAction(Integrate.java:136)
        at org.intermine.task.Integrate.execute(Integrate.java:127)
        at org.apache.tools.ant.UnknownElement.execute(UnknownElement.java:291)
        at sun.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:597)
        at org.apache.tools.ant.dispatch.DispatchUtils.execute(DispatchUtils.java:106)
        at org.apache.tools.ant.Task.perform(Task.java:348)
        at org.apache.tools.ant.Target.execute(Target.java:390)
        at org.apache.tools.ant.Target.performTasks(Target.java:411)
        at org.apache.tools.ant.Project.executeSortedTargets(Project.java:1360)
        at org.apache.tools.ant.Project.executeTarget(Project.java:1329)
        at org.apache.tools.ant.helper.DefaultExecutor.executeTargets(DefaultExecutor.java:41)
        at org.apache.tools.ant.Project.executeTargets(Project.java:1212)
        at org.apache.tools.ant.Main.runBuild(Main.java:801)
        at org.apache.tools.ant.Main.startAnt(Main.java:218)
        at org.apache.tools.ant.launch.Launcher.run(Launcher.java:280)
        at org.apache.tools.ant.launch.Launcher.main(Launcher.java:109)
Caused by: java.lang.reflect.InvocationTargetException
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
        at java.lang.reflect.Method.invoke(Method.java:597)
        at org.intermine.dataloader.IntegrationWriterFactory.getIntegrationWriter(IntegrationWriterFactory.java:65)
        ... 34 more
Caused by: org.intermine.objectstore.ObjectStoreException: Failed to instantiate ObjectStoreWriter class: org.intermine.objectstore.intermine.ObjectStoreWriterInterMineImpl, osAlias: os.production - java.lang.RuntimeException: can't open datasource for {platform=PostgreSQL, datasource.dataSourceName=db.production, datasource.maxConnections=50, datasource.databaseName=wormmine-test-t, datasource.class=org.postgresql.ds.PGPoolingDataSource, datasource.password=#####, datasource.user=intermine, driver=org.postgresql.Driver, datasource.serverName=localhost} org.postgresql.Driver PostgreSQL
        at org.intermine.objectstore.ObjectStoreWriterFactory.getObjectStoreWriter(ObjectStoreWriterFactory.java:89)
        at org.intermine.dataloader.IntegrationWriterDataTrackingImpl.getInstance(IntegrationWriterDataTrackingImpl.java:120)
        at org.intermine.dataloader.IntegrationWriterDataTrackingImpl.getInstance(IntegrationWriterDataTrackingImpl.java:80)
        ... 39 more
Caused by: java.lang.RuntimeException: can't open datasource for {platform=PostgreSQL, datasource.dataSourceName=db.production, datasource.maxConnections=50, datasource.databaseName=wormmine-test-t, datasource.class=org.postgresql.ds.PGPoolingDataSource, datasource.password=#####, datasource.user=intermine, driver=org.postgresql.Driver, datasource.serverName=localhost} org.postgresql.Driver PostgreSQL
        at org.intermine.sql.Database.getConnection(Database.java:106)
        at org.intermine.objectstore.intermine.ObjectStoreInterMineImpl.getConnection(ObjectStoreInterMineImpl.java:256)
        at org.intermine.objectstore.intermine.ObjectStoreWriterInterMineImpl.<init>(ObjectStoreWriterInterMineImpl.java:117)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
        at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:513)
        at org.intermine.objectstore.ObjectStoreWriterFactory.getObjectStoreWriter(ObjectStoreWriterFactory.java:78)
        ... 41 more
Caused by: org.postgresql.util.PSQLException: FATAL: connection limit exceeded for non-superusers
        at org.postgresql.core.v3.ConnectionFactoryImpl.readStartupMessages(ConnectionFactoryImpl.java:464)
        at org.postgresql.core.v3.ConnectionFactoryImpl.openConnectionImpl(ConnectionFactoryImpl.java:112)
        at org.postgresql.core.ConnectionFactory.openConnection(ConnectionFactory.java:66)
        at org.postgresql.jdbc2.AbstractJdbc2Connection.<init>(AbstractJdbc2Connection.java:125)
        at org.postgresql.jdbc3.AbstractJdbc3Connection.<init>(AbstractJdbc3Connection.java:30)
        at org.postgresql.jdbc3g.AbstractJdbc3gConnection.<init>(AbstractJdbc3gConnection.java:22)
        at org.postgresql.jdbc3g.Jdbc3gConnection.<init>(Jdbc3gConnection.java:24)
        at org.postgresql.Driver.makeConnection(Driver.java:393)
        at org.postgresql.Driver.connect(Driver.java:267)
        at java.sql.DriverManager.getConnection(DriverManager.java:582)
        at java.sql.DriverManager.getConnection(DriverManager.java:185)
        at org.postgresql.ds.common.BaseDataSource.getConnection(BaseDataSource.java:91)
        at org.postgresql.ds.common.BaseDataSource.getConnection(BaseDataSource.java:74)
        at org.postgresql.ds.PGConnectionPoolDataSource.getPooledConnection(PGConnectionPoolDataSource.java:60)
        at org.postgresql.ds.jdbc23.AbstractJdbc23PoolingDataSource.getPooledConnection(AbstractJdbc23PoolingDataSource.java:429)
        at org.postgresql.ds.jdbc23.AbstractJdbc23PoolingDataSource.getConnection(AbstractJdbc23PoolingDataSource.java:356)
        at org.intermine.sql.Database.getConnection(Database.java:104)
        ... 48 more


_______________________________________________
dev mailing list
[hidden email]
http://mail.intermine.org/cgi-bin/mailman/listinfo/dev
Reply | Threaded
Open this post in threaded view
|

Re: Overloading postgres on build

Julie Sullivan
JD

Can you verify that you are using the project_build script? In FlyMine
we have 38 FASTA sources and don't have any problems:

https://github.com/intermine/intermine/blob/dev/flymine/project.xml#L173

So your build should be fine!

There is a maxConnections parameter for each one in either

        default.intermine.integrate.properties
        default.intermine.webapp.properties

You can increase the value and see if it helps?

If it doesn't it's probably the postgres installation itself running out
of connections, in which case you could check whether you have lots
of webapps deployed.  There are also postgres settings you can change
for altering the number of available connections, I can point these out
if you're still having problems.

Julie

On 17/09/13 00:05, JD Wong wrote:

> Hi all,
>
> It's been a while since the last help request, and so it's time to break
> that trend.
>
> I've recently expanded my build to include the genomic and protein fasta
> sources for all the species in wormbase along the lines of 48 fasta
> based sources .
>
> Loading them, I get this new stack trace.
>
> It seems that the mine might be opening a new database connection for
> each source.  Can anybody comment on the root cause & how to fix it?
>
> Thanks!
>
> BUILD FAILED
> /nfs/wormbase/wormmine/website-intermine-dev/acedb-dev/intermine/imbuild/integrate.xml:54:
> The following error occurred while executing this line:
> /nfs/wormbase/wormmine/website-intermine-dev/acedb-dev/intermine/bio/sources/fasta/build.xml:46:
> org.intermine.objectstore.ObjectStoreException: Failed to instantiate
> IntegrationWriter class:
> org.intermine.dataloader.IntegrationWriterDataTrackingImpl
>          at
> org.intermine.dataloader.IntegrationWriterFactory.getIntegrationWriter(IntegrationWriterFactory.java:75)
>          at
> org.intermine.task.DirectDataLoaderTask.getIntegrationWriter(DirectDataLoaderTask.java:85)
>          at
> org.intermine.task.DirectDataLoaderTask.execute(DirectDataLoaderTask.java:129)
>          at
> org.intermine.bio.dataconversion.FastaLoaderTask.execute(FastaLoaderTask.java:200)
>          at
> org.apache.tools.ant.UnknownElement.execute(UnknownElement.java:291)
>          at sun.reflect.GeneratedMethodAccessor360.invoke(Unknown Source)
>          at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>          at java.lang.reflect.Method.invoke(Method.java:597)
>          at
> org.apache.tools.ant.dispatch.DispatchUtils.execute(DispatchUtils.java:106)
>          at org.apache.tools.ant.Task.perform(Task.java:348)
>          at org.apache.tools.ant.Target.execute(Target.java:390)
>          at org.apache.tools.ant.Target.performTasks(Target.java:411)
>          at
> org.apache.tools.ant.Project.executeSortedTargets(Project.java:1360)
>          at
> org.apache.tools.ant.helper.SingleCheckExecutor.executeTargets(SingleCheckExecutor.java:38)
>          at org.apache.tools.ant.Project.executeTargets(Project.java:1212)
>          at org.apache.tools.ant.taskdefs.Ant.execute(Ant.java:441)
>          at org.intermine.task.Integrate.performAction(Integrate.java:223)
>          at org.intermine.task.Integrate.performAction(Integrate.java:136)
>          at org.intermine.task.Integrate.execute(Integrate.java:127)
>          at
> org.apache.tools.ant.UnknownElement.execute(UnknownElement.java:291)
>          at sun.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
>          at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>          at java.lang.reflect.Method.invoke(Method.java:597)
>          at
> org.apache.tools.ant.dispatch.DispatchUtils.execute(DispatchUtils.java:106)
>          at org.apache.tools.ant.Task.perform(Task.java:348)
>          at org.apache.tools.ant.Target.execute(Target.java:390)
>          at org.apache.tools.ant.Target.performTasks(Target.java:411)
>          at
> org.apache.tools.ant.Project.executeSortedTargets(Project.java:1360)
>          at org.apache.tools.ant.Project.executeTarget(Project.java:1329)
>          at
> org.apache.tools.ant.helper.DefaultExecutor.executeTargets(DefaultExecutor.java:41)
>          at org.apache.tools.ant.Project.executeTargets(Project.java:1212)
>          at org.apache.tools.ant.Main.runBuild(Main.java:801)
>          at org.apache.tools.ant.Main.startAnt(Main.java:218)
>          at org.apache.tools.ant.launch.Launcher.run(Launcher.java:280)
>          at org.apache.tools.ant.launch.Launcher.main(Launcher.java:109)
> Caused by: java.lang.reflect.InvocationTargetException
>          at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>          at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
>          at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
>          at java.lang.reflect.Method.invoke(Method.java:597)
>          at
> org.intermine.dataloader.IntegrationWriterFactory.getIntegrationWriter(IntegrationWriterFactory.java:65)
>          ... 34 more
> Caused by: org.intermine.objectstore.ObjectStoreException: Failed to
> instantiate ObjectStoreWriter class:
> org.intermine.objectstore.intermine.ObjectStoreWriterInterMineImpl,
> osAlias: os.production - java.lang.RuntimeException: can't open
> datasource for {platform=PostgreSQL,
> datasource.dataSourceName=db.production, datasource.maxConnections=50,
> datasource.databaseName=wormmine-test-t,
> datasource.class=org.postgresql.ds.PGPoolingDataSource,
> datasource.password=#####, datasource.user=intermine,
> driver=org.postgresql.Driver, datasource.serverName=localhost}
> org.postgresql.Driver PostgreSQL
>          at
> org.intermine.objectstore.ObjectStoreWriterFactory.getObjectStoreWriter(ObjectStoreWriterFactory.java:89)
>          at
> org.intermine.dataloader.IntegrationWriterDataTrackingImpl.getInstance(IntegrationWriterDataTrackingImpl.java:120)
>          at
> org.intermine.dataloader.IntegrationWriterDataTrackingImpl.getInstance(IntegrationWriterDataTrackingImpl.java:80)
>          ... 39 more
> Caused by: java.lang.RuntimeException: can't open datasource for
> {platform=PostgreSQL, datasource.dataSourceName=db.production,
> datasource.maxConnections=50, datasource.databaseName=wormmine-test-t,
> datasource.class=org.postgresql.ds.PGPoolingDataSource,
> datasource.password=#####, datasource.user=intermine,
> driver=org.postgresql.Driver, datasource.serverName=localhost}
> org.postgresql.Driver PostgreSQL
>          at org.intermine.sql.Database.getConnection(Database.java:106)
>          at
> org.intermine.objectstore.intermine.ObjectStoreInterMineImpl.getConnection(ObjectStoreInterMineImpl.java:256)
>          at
> org.intermine.objectstore.intermine.ObjectStoreWriterInterMineImpl.<init>(ObjectStoreWriterInterMineImpl.java:117)
>          at
> sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
>          at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:39)
>          at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:27)
>          at java.lang.reflect.Constructor.newInstance(Constructor.java:513)
>          at
> org.intermine.objectstore.ObjectStoreWriterFactory.getObjectStoreWriter(ObjectStoreWriterFactory.java:78)
>          ... 41 more
> Caused by: org.postgresql.util.PSQLException: FATAL: connection limit
> exceeded for non-superusers
>          at
> org.postgresql.core.v3.ConnectionFactoryImpl.readStartupMessages(ConnectionFactoryImpl.java:464)
>          at
> org.postgresql.core.v3.ConnectionFactoryImpl.openConnectionImpl(ConnectionFactoryImpl.java:112)
>          at
> org.postgresql.core.ConnectionFactory.openConnection(ConnectionFactory.java:66)
>          at
> org.postgresql.jdbc2.AbstractJdbc2Connection.<init>(AbstractJdbc2Connection.java:125)
>          at
> org.postgresql.jdbc3.AbstractJdbc3Connection.<init>(AbstractJdbc3Connection.java:30)
>          at
> org.postgresql.jdbc3g.AbstractJdbc3gConnection.<init>(AbstractJdbc3gConnection.java:22)
>          at
> org.postgresql.jdbc3g.Jdbc3gConnection.<init>(Jdbc3gConnection.java:24)
>          at org.postgresql.Driver.makeConnection(Driver.java:393)
>          at org.postgresql.Driver.connect(Driver.java:267)
>          at java.sql.DriverManager.getConnection(DriverManager.java:582)
>          at java.sql.DriverManager.getConnection(DriverManager.java:185)
>          at
> org.postgresql.ds.common.BaseDataSource.getConnection(BaseDataSource.java:91)
>          at
> org.postgresql.ds.common.BaseDataSource.getConnection(BaseDataSource.java:74)
>          at
> org.postgresql.ds.PGConnectionPoolDataSource.getPooledConnection(PGConnectionPoolDataSource.java:60)
>          at
> org.postgresql.ds.jdbc23.AbstractJdbc23PoolingDataSource.getPooledConnection(AbstractJdbc23PoolingDataSource.java:429)
>          at
> org.postgresql.ds.jdbc23.AbstractJdbc23PoolingDataSource.getConnection(AbstractJdbc23PoolingDataSource.java:356)
>          at org.intermine.sql.Database.getConnection(Database.java:104)
>          ... 48 more
>
>
>
> _______________________________________________
> dev mailing list
> [hidden email]
> http://mail.intermine.org/cgi-bin/mailman/listinfo/dev
>

_______________________________________________
dev mailing list
[hidden email]
http://mail.intermine.org/cgi-bin/mailman/listinfo/dev
Reply | Threaded
Open this post in threaded view
|

Re: Overloading postgres on build

Julie Sullivan
Yes, that's it! Unfortunately, ant isolates each individual task in its
own ClassLoader, which prevents the different tasks from sharing
database connections. Each ClassLoader will end up having its own
connection pool, with a maximum number of connections as set in the
properties. Moreover, it doesn't tend to allow those ClassLoaders or
database connection pools to be garbage collected once the task has
finished. Therefore, each individual task will leave several idle
database connections lying around. We tried to work around this problem
a while back too, but were unsuccessful. Ant simply isolates the
individual tasks too well.

On 17/09/13 23:10, JD Wong wrote:

> I called each source directly through the Dsource parameter because I
> only wanted to run the new sources.  Hopefully this is the cause of the
> problem!
>
> Thanks,
> -JD
>
>
> On Tue, Sep 17, 2013 at 6:34 AM, Julie Sullivan <[hidden email]
> <mailto:[hidden email]>> wrote:
>
>     JD
>
>     Can you verify that you are using the project_build script? In
>     FlyMine we have 38 FASTA sources and don't have any problems:
>
>     https://github.com/intermine/__intermine/blob/dev/flymine/__project.xml#L173
>     <https://github.com/intermine/intermine/blob/dev/flymine/project.xml#L173>
>
>     So your build should be fine!
>
>     There is a maxConnections parameter for each one in either
>
>              default.intermine.integrate.__properties
>              default.intermine.webapp.__properties
>
>     You can increase the value and see if it helps?
>
>     If it doesn't it's probably the postgres installation itself running out
>     of connections, in which case you could check whether you have lots
>     of webapps deployed.  There are also postgres settings you can
>     change for altering the number of available connections, I can point
>     these out if you're still having problems.
>
>     Julie
>
>
>     On 17/09/13 00:05, JD Wong wrote:
>
>         Hi all,
>
>         It's been a while since the last help request, and so it's time
>         to break
>         that trend.
>
>         I've recently expanded my build to include the genomic and
>         protein fasta
>         sources for all the species in wormbase along the lines of 48 fasta
>         based sources .
>
>         Loading them, I get this new stack trace.
>
>         It seems that the mine might be opening a new database
>         connection for
>         each source.  Can anybody comment on the root cause & how to fix it?
>
>         Thanks!
>
>         BUILD FAILED
>         /nfs/wormbase/wormmine/__website-intermine-dev/acedb-__dev/intermine/imbuild/__integrate.xml:54:
>         The following error occurred while executing this line:
>         /nfs/wormbase/wormmine/__website-intermine-dev/acedb-__dev/intermine/bio/sources/__fasta/build.xml:46:
>         org.intermine.objectstore.__ObjectStoreException: Failed to
>         instantiate
>         IntegrationWriter class:
>         org.intermine.dataloader.__IntegrationWriterDataTrackingI__mpl
>                   at
>         org.intermine.dataloader.__IntegrationWriterFactory.__getIntegrationWriter(__IntegrationWriterFactory.java:__75)
>                   at
>         org.intermine.task.__DirectDataLoaderTask.__getIntegrationWriter(__DirectDataLoaderTask.java:85)
>                   at
>         org.intermine.task.__DirectDataLoaderTask.execute(__DirectDataLoaderTask.java:129)
>                   at
>         org.intermine.bio.__dataconversion.__FastaLoaderTask.execute(__FastaLoaderTask.java:200)
>                   at
>         org.apache.tools.ant.__UnknownElement.execute(__UnknownElement.java:291)
>                   at
>         sun.reflect.__GeneratedMethodAccessor360.__invoke(Unknown Source)
>                   at
>         sun.reflect.__DelegatingMethodAccessorImpl.__invoke(__DelegatingMethodAccessorImpl.__java:25)
>                   at java.lang.reflect.Method.__invoke(Method.java:597)
>                   at
>         org.apache.tools.ant.dispatch.__DispatchUtils.execute(__DispatchUtils.java:106)
>                   at org.apache.tools.ant.Task.__perform(Task.java:348)
>                   at org.apache.tools.ant.Target.__execute(Target.java:390)
>                   at
>         org.apache.tools.ant.Target.__performTasks(Target.java:411)
>                   at
>         org.apache.tools.ant.Project.__executeSortedTargets(Project.__java:1360)
>                   at
>         org.apache.tools.ant.helper.__SingleCheckExecutor.__executeTargets(__SingleCheckExecutor.java:38)
>                   at
>         org.apache.tools.ant.Project.__executeTargets(Project.java:__1212)
>                   at
>         org.apache.tools.ant.taskdefs.__Ant.execute(Ant.java:441)
>                   at
>         org.intermine.task.Integrate.__performAction(Integrate.java:__223)
>                   at
>         org.intermine.task.Integrate.__performAction(Integrate.java:__136)
>                   at
>         org.intermine.task.Integrate.__execute(Integrate.java:127)
>                   at
>         org.apache.tools.ant.__UnknownElement.execute(__UnknownElement.java:291)
>                   at
>         sun.reflect.__GeneratedMethodAccessor4.__invoke(Unknown Source)
>                   at
>         sun.reflect.__DelegatingMethodAccessorImpl.__invoke(__DelegatingMethodAccessorImpl.__java:25)
>                   at java.lang.reflect.Method.__invoke(Method.java:597)
>                   at
>         org.apache.tools.ant.dispatch.__DispatchUtils.execute(__DispatchUtils.java:106)
>                   at org.apache.tools.ant.Task.__perform(Task.java:348)
>                   at org.apache.tools.ant.Target.__execute(Target.java:390)
>                   at
>         org.apache.tools.ant.Target.__performTasks(Target.java:411)
>                   at
>         org.apache.tools.ant.Project.__executeSortedTargets(Project.__java:1360)
>                   at
>         org.apache.tools.ant.Project.__executeTarget(Project.java:__1329)
>                   at
>         org.apache.tools.ant.helper.__DefaultExecutor.__executeTargets(__DefaultExecutor.java:41)
>                   at
>         org.apache.tools.ant.Project.__executeTargets(Project.java:__1212)
>                   at org.apache.tools.ant.Main.__runBuild(Main.java:801)
>                   at org.apache.tools.ant.Main.__startAnt(Main.java:218)
>                   at
>         org.apache.tools.ant.launch.__Launcher.run(Launcher.java:__280)
>                   at
>         org.apache.tools.ant.launch.__Launcher.main(Launcher.java:__109)
>         Caused by: java.lang.reflect.__InvocationTargetException
>                   at
>         sun.reflect.__NativeMethodAccessorImpl.__invoke0(Native Method)
>                   at
>         sun.reflect.__NativeMethodAccessorImpl.__invoke(__NativeMethodAccessorImpl.java:__39)
>                   at
>         sun.reflect.__DelegatingMethodAccessorImpl.__invoke(__DelegatingMethodAccessorImpl.__java:25)
>                   at java.lang.reflect.Method.__invoke(Method.java:597)
>                   at
>         org.intermine.dataloader.__IntegrationWriterFactory.__getIntegrationWriter(__IntegrationWriterFactory.java:__65)
>                   ... 34 more
>         Caused by: org.intermine.objectstore.__ObjectStoreException:
>         Failed to
>         instantiate ObjectStoreWriter class:
>         org.intermine.objectstore.__intermine.__ObjectStoreWriterInterMineImpl__,
>         osAlias: os.production - java.lang.RuntimeException: can't open
>         datasource for {platform=PostgreSQL,
>         datasource.dataSourceName=db.__production,
>         datasource.maxConnections=50,
>         datasource.databaseName=__wormmine-test-t,
>         datasource.class=org.__postgresql.ds.__PGPoolingDataSource,
>         datasource.password=#####, datasource.user=intermine,
>         driver=org.postgresql.Driver, datasource.serverName=__localhost}
>         org.postgresql.Driver PostgreSQL
>                   at
>         org.intermine.objectstore.__ObjectStoreWriterFactory.__getObjectStoreWriter(__ObjectStoreWriterFactory.java:__89)
>                   at
>         org.intermine.dataloader.__IntegrationWriterDataTrackingI__mpl.getInstance(__IntegrationWriterDataTrackingI__mpl.java:120)
>                   at
>         org.intermine.dataloader.__IntegrationWriterDataTrackingI__mpl.getInstance(__IntegrationWriterDataTrackingI__mpl.java:80)
>                   ... 39 more
>         Caused by: java.lang.RuntimeException: can't open datasource for
>         {platform=PostgreSQL, datasource.dataSourceName=db.__production,
>         datasource.maxConnections=50,
>         datasource.databaseName=__wormmine-test-t,
>         datasource.class=org.__postgresql.ds.__PGPoolingDataSource,
>         datasource.password=#####, datasource.user=intermine,
>         driver=org.postgresql.Driver, datasource.serverName=__localhost}
>         org.postgresql.Driver PostgreSQL
>                   at
>         org.intermine.sql.Database.__getConnection(Database.java:__106)
>                   at
>         org.intermine.objectstore.__intermine.__ObjectStoreInterMineImpl.__getConnection(__ObjectStoreInterMineImpl.java:__256)
>                   at
>         org.intermine.objectstore.__intermine.__ObjectStoreWriterInterMineImpl__.<init>(__ObjectStoreWriterInterMineImpl__.java:117)
>                   at
>         sun.reflect.__NativeConstructorAccessorImpl.__newInstance0(Native Method)
>                   at
>         sun.reflect.__NativeConstructorAccessorImpl.__newInstance(__NativeConstructorAccessorImpl.__java:39)
>                   at
>         sun.reflect.__DelegatingConstructorAccessorI__mpl.newInstance(__DelegatingConstructorAccessorI__mpl.java:27)
>                   at
>         java.lang.reflect.Constructor.__newInstance(Constructor.java:__513)
>                   at
>         org.intermine.objectstore.__ObjectStoreWriterFactory.__getObjectStoreWriter(__ObjectStoreWriterFactory.java:__78)
>                   ... 41 more
>         Caused by: org.postgresql.util.__PSQLException: FATAL:
>         connection limit
>         exceeded for non-superusers
>                   at
>         org.postgresql.core.v3.__ConnectionFactoryImpl.__readStartupMessages(__ConnectionFactoryImpl.java:__464)
>                   at
>         org.postgresql.core.v3.__ConnectionFactoryImpl.__openConnectionImpl(__ConnectionFactoryImpl.java:__112)
>                   at
>         org.postgresql.core.__ConnectionFactory.__openConnection(__ConnectionFactory.java:66)
>                   at
>         org.postgresql.jdbc2.__AbstractJdbc2Connection.<init>__(AbstractJdbc2Connection.java:__125)
>                   at
>         org.postgresql.jdbc3.__AbstractJdbc3Connection.<init>__(AbstractJdbc3Connection.java:__30)
>                   at
>         org.postgresql.jdbc3g.__AbstractJdbc3gConnection.<__init>(__AbstractJdbc3gConnection.java:__22)
>                   at
>         org.postgresql.jdbc3g.__Jdbc3gConnection.<init>(__Jdbc3gConnection.java:24)
>                   at
>         org.postgresql.Driver.__makeConnection(Driver.java:__393)
>                   at org.postgresql.Driver.connect(__Driver.java:267)
>                   at
>         java.sql.DriverManager.__getConnection(DriverManager.__java:582)
>                   at
>         java.sql.DriverManager.__getConnection(DriverManager.__java:185)
>                   at
>         org.postgresql.ds.common.__BaseDataSource.getConnection(__BaseDataSource.java:91)
>                   at
>         org.postgresql.ds.common.__BaseDataSource.getConnection(__BaseDataSource.java:74)
>                   at
>         org.postgresql.ds.__PGConnectionPoolDataSource.__getPooledConnection(__PGConnectionPoolDataSource.__java:60)
>                   at
>         org.postgresql.ds.jdbc23.__AbstractJdbc23PoolingDataSourc__e.getPooledConnection(__AbstractJdbc23PoolingDataSourc__e.java:429)
>                   at
>         org.postgresql.ds.jdbc23.__AbstractJdbc23PoolingDataSourc__e.getConnection(__AbstractJdbc23PoolingDataSourc__e.java:356)
>                   at
>         org.intermine.sql.Database.__getConnection(Database.java:__104)
>                   ... 48 more
>
>
>
>         _________________________________________________
>         dev mailing list
>         [hidden email] <mailto:[hidden email]>
>         http://mail.intermine.org/cgi-__bin/mailman/listinfo/dev
>         <http://mail.intermine.org/cgi-bin/mailman/listinfo/dev>
>
>

_______________________________________________
dev mailing list
[hidden email]
http://mail.intermine.org/cgi-bin/mailman/listinfo/dev
Reply | Threaded
Open this post in threaded view
|

Re: Overloading postgres on build

JD Wong
Thanks for the helpful explanation!  So it looks like the best option is to create a decoy project.xml containing only my desired source, then running the project_build script on it, unless there is some other way I haven't thought of.

Cheers,
- JD


On Wed, Sep 18, 2013 at 4:47 AM, Julie Sullivan <[hidden email]> wrote:
Yes, that's it! Unfortunately, ant isolates each individual task in its own ClassLoader, which prevents the different tasks from sharing database connections. Each ClassLoader will end up having its own connection pool, with a maximum number of connections as set in the properties. Moreover, it doesn't tend to allow those ClassLoaders or database connection pools to be garbage collected once the task has finished. Therefore, each individual task will leave several idle database connections lying around. We tried to work around this problem a while back too, but were unsuccessful. Ant simply isolates the individual tasks too well.


On 17/09/13 23:10, JD Wong wrote:
I called each source directly through the Dsource parameter because I
only wanted to run the new sources.  Hopefully this is the cause of the
problem!

Thanks,
-JD


On Tue, Sep 17, 2013 at 6:34 AM, Julie Sullivan <[hidden email]
<mailto:[hidden email]>> wrote:

    JD

    Can you verify that you are using the project_build script? In
    FlyMine we have 38 FASTA sources and don't have any problems:

    https://github.com/intermine/__intermine/blob/dev/flymine/__project.xml#L173

    <https://github.com/intermine/intermine/blob/dev/flymine/project.xml#L173>

    So your build should be fine!

    There is a maxConnections parameter for each one in either

             default.intermine.integrate.__properties
             default.intermine.webapp.__properties


    You can increase the value and see if it helps?

    If it doesn't it's probably the postgres installation itself running out
    of connections, in which case you could check whether you have lots
    of webapps deployed.  There are also postgres settings you can
    change for altering the number of available connections, I can point
    these out if you're still having problems.

    Julie


    On 17/09/13 00:05, JD Wong wrote:

        Hi all,

        It's been a while since the last help request, and so it's time
        to break
        that trend.

        I've recently expanded my build to include the genomic and
        protein fasta
        sources for all the species in wormbase along the lines of 48 fasta
        based sources .

        Loading them, I get this new stack trace.

        It seems that the mine might be opening a new database
        connection for
        each source.  Can anybody comment on the root cause & how to fix it?

        Thanks!

        BUILD FAILED
        /nfs/wormbase/wormmine/__website-intermine-dev/acedb-__dev/intermine/imbuild/__integrate.xml:54:

        The following error occurred while executing this line:
        /nfs/wormbase/wormmine/__website-intermine-dev/acedb-__dev/intermine/bio/sources/__fasta/build.xml:46:
        org.intermine.objectstore.__ObjectStoreException: Failed to
        instantiate
        IntegrationWriter class:
        org.intermine.dataloader.__IntegrationWriterDataTrackingI__mpl
                  at
        org.intermine.dataloader.__IntegrationWriterFactory.__getIntegrationWriter(__IntegrationWriterFactory.java:__75)
                  at
        org.intermine.task.__DirectDataLoaderTask.__getIntegrationWriter(__DirectDataLoaderTask.java:85)
                  at
        org.intermine.task.__DirectDataLoaderTask.execute(__DirectDataLoaderTask.java:129)
                  at
        org.intermine.bio.__dataconversion.__FastaLoaderTask.execute(__FastaLoaderTask.java:200)
                  at
        org.apache.tools.ant.__UnknownElement.execute(__UnknownElement.java:291)
                  at
        sun.reflect.__GeneratedMethodAccessor360.__invoke(Unknown Source)
                  at
        sun.reflect.__DelegatingMethodAccessorImpl.__invoke(__DelegatingMethodAccessorImpl.__java:25)
                  at java.lang.reflect.Method.__invoke(Method.java:597)
                  at
        org.apache.tools.ant.dispatch.__DispatchUtils.execute(__DispatchUtils.java:106)
                  at org.apache.tools.ant.Task.__perform(Task.java:348)
                  at org.apache.tools.ant.Target.__execute(Target.java:390)
                  at
        org.apache.tools.ant.Target.__performTasks(Target.java:411)
                  at
        org.apache.tools.ant.Project.__executeSortedTargets(Project.__java:1360)
                  at
        org.apache.tools.ant.helper.__SingleCheckExecutor.__executeTargets(__SingleCheckExecutor.java:38)
                  at
        org.apache.tools.ant.Project.__executeTargets(Project.java:__1212)
                  at
        org.apache.tools.ant.taskdefs.__Ant.execute(Ant.java:441)
                  at
        org.intermine.task.Integrate.__performAction(Integrate.java:__223)
                  at
        org.intermine.task.Integrate.__performAction(Integrate.java:__136)
                  at
        org.intermine.task.Integrate.__execute(Integrate.java:127)
                  at
        org.apache.tools.ant.__UnknownElement.execute(__UnknownElement.java:291)
                  at
        sun.reflect.__GeneratedMethodAccessor4.__invoke(Unknown Source)
                  at
        sun.reflect.__DelegatingMethodAccessorImpl.__invoke(__DelegatingMethodAccessorImpl.__java:25)
                  at java.lang.reflect.Method.__invoke(Method.java:597)
                  at
        org.apache.tools.ant.dispatch.__DispatchUtils.execute(__DispatchUtils.java:106)
                  at org.apache.tools.ant.Task.__perform(Task.java:348)
                  at org.apache.tools.ant.Target.__execute(Target.java:390)
                  at
        org.apache.tools.ant.Target.__performTasks(Target.java:411)
                  at
        org.apache.tools.ant.Project.__executeSortedTargets(Project.__java:1360)
                  at
        org.apache.tools.ant.Project.__executeTarget(Project.java:__1329)
                  at
        org.apache.tools.ant.helper.__DefaultExecutor.__executeTargets(__DefaultExecutor.java:41)
                  at
        org.apache.tools.ant.Project.__executeTargets(Project.java:__1212)
                  at org.apache.tools.ant.Main.__runBuild(Main.java:801)
                  at org.apache.tools.ant.Main.__startAnt(Main.java:218)
                  at
        org.apache.tools.ant.launch.__Launcher.run(Launcher.java:__280)
                  at
        org.apache.tools.ant.launch.__Launcher.main(Launcher.java:__109)
        Caused by: java.lang.reflect.__InvocationTargetException
                  at
        sun.reflect.__NativeMethodAccessorImpl.__invoke0(Native Method)
                  at
        sun.reflect.__NativeMethodAccessorImpl.__invoke(__NativeMethodAccessorImpl.java:__39)
                  at
        sun.reflect.__DelegatingMethodAccessorImpl.__invoke(__DelegatingMethodAccessorImpl.__java:25)
                  at java.lang.reflect.Method.__invoke(Method.java:597)
                  at
        org.intermine.dataloader.__IntegrationWriterFactory.__getIntegrationWriter(__IntegrationWriterFactory.java:__65)
                  ... 34 more
        Caused by: org.intermine.objectstore.__ObjectStoreException:

        Failed to
        instantiate ObjectStoreWriter class:
        org.intermine.objectstore.__intermine.__ObjectStoreWriterInterMineImpl__,

        osAlias: os.production - java.lang.RuntimeException: can't open
        datasource for {platform=PostgreSQL,
        datasource.dataSourceName=db.__production,
        datasource.maxConnections=50,
        datasource.databaseName=__wormmine-test-t,
        datasource.class=org.__postgresql.ds.__PGPoolingDataSource,
        datasource.password=#####, datasource.user=intermine,
        driver=org.postgresql.Driver, datasource.serverName=__localhost}
        org.postgresql.Driver PostgreSQL
                  at
        org.intermine.objectstore.__ObjectStoreWriterFactory.__getObjectStoreWriter(__ObjectStoreWriterFactory.java:__89)
                  at
        org.intermine.dataloader.__IntegrationWriterDataTrackingI__mpl.getInstance(__IntegrationWriterDataTrackingI__mpl.java:120)
                  at
        org.intermine.dataloader.__IntegrationWriterDataTrackingI__mpl.getInstance(__IntegrationWriterDataTrackingI__mpl.java:80)

                  ... 39 more
        Caused by: java.lang.RuntimeException: can't open datasource for
        {platform=PostgreSQL, datasource.dataSourceName=db.__production,
        datasource.maxConnections=50,
        datasource.databaseName=__wormmine-test-t,
        datasource.class=org.__postgresql.ds.__PGPoolingDataSource,
        datasource.password=#####, datasource.user=intermine,
        driver=org.postgresql.Driver, datasource.serverName=__localhost}
        org.postgresql.Driver PostgreSQL
                  at
        org.intermine.sql.Database.__getConnection(Database.java:__106)
                  at
        org.intermine.objectstore.__intermine.__ObjectStoreInterMineImpl.__getConnection(__ObjectStoreInterMineImpl.java:__256)
                  at
        org.intermine.objectstore.__intermine.__ObjectStoreWriterInterMineImpl__.<init>(__ObjectStoreWriterInterMineImpl__.java:117)
                  at
        sun.reflect.__NativeConstructorAccessorImpl.__newInstance0(Native Method)
                  at
        sun.reflect.__NativeConstructorAccessorImpl.__newInstance(__NativeConstructorAccessorImpl.__java:39)
                  at
        sun.reflect.__DelegatingConstructorAccessorI__mpl.newInstance(__DelegatingConstructorAccessorI__mpl.java:27)
                  at
        java.lang.reflect.Constructor.__newInstance(Constructor.java:__513)
                  at
        org.intermine.objectstore.__ObjectStoreWriterFactory.__getObjectStoreWriter(__ObjectStoreWriterFactory.java:__78)
                  ... 41 more
        Caused by: org.postgresql.util.__PSQLException: FATAL:

        connection limit
        exceeded for non-superusers
                  at
        org.postgresql.core.v3.__ConnectionFactoryImpl.__readStartupMessages(__ConnectionFactoryImpl.java:__464)
                  at
        org.postgresql.core.v3.__ConnectionFactoryImpl.__openConnectionImpl(__ConnectionFactoryImpl.java:__112)
                  at
        org.postgresql.core.__ConnectionFactory.__openConnection(__ConnectionFactory.java:66)
                  at
        org.postgresql.jdbc2.__AbstractJdbc2Connection.<init>__(AbstractJdbc2Connection.java:__125)
                  at
        org.postgresql.jdbc3.__AbstractJdbc3Connection.<init>__(AbstractJdbc3Connection.java:__30)
                  at
        org.postgresql.jdbc3g.__AbstractJdbc3gConnection.<__init>(__AbstractJdbc3gConnection.java:__22)
                  at
        org.postgresql.jdbc3g.__Jdbc3gConnection.<init>(__Jdbc3gConnection.java:24)
                  at
        org.postgresql.Driver.__makeConnection(Driver.java:__393)
                  at org.postgresql.Driver.connect(__Driver.java:267)
                  at
        java.sql.DriverManager.__getConnection(DriverManager.__java:582)
                  at
        java.sql.DriverManager.__getConnection(DriverManager.__java:185)
                  at
        org.postgresql.ds.common.__BaseDataSource.getConnection(__BaseDataSource.java:91)
                  at
        org.postgresql.ds.common.__BaseDataSource.getConnection(__BaseDataSource.java:74)
                  at
        org.postgresql.ds.__PGConnectionPoolDataSource.__getPooledConnection(__PGConnectionPoolDataSource.__java:60)
                  at
        org.postgresql.ds.jdbc23.__AbstractJdbc23PoolingDataSourc__e.getPooledConnection(__AbstractJdbc23PoolingDataSourc__e.java:429)
                  at
        org.postgresql.ds.jdbc23.__AbstractJdbc23PoolingDataSourc__e.getConnection(__AbstractJdbc23PoolingDataSourc__e.java:356)
                  at
        org.intermine.sql.Database.__getConnection(Database.java:__104)
                  ... 48 more



        _________________________________________________
        dev mailing list
        [hidden email] <mailto:[hidden email]>
        http://mail.intermine.org/cgi-__bin/mailman/listinfo/dev
        <http://mail.intermine.org/cgi-bin/mailman/listinfo/dev>




_______________________________________________
dev mailing list
[hidden email]
http://mail.intermine.org/cgi-bin/mailman/listinfo/dev
Reply | Threaded
Open this post in threaded view
|

Re: Overloading postgres on build

Julie Sullivan
Yes, I suppose that's the best way, although that doesn't seem great!

Would it be useful to be able to pass the source names you want to run
to the project_build script? Or maybe be able to specify different
project XML files?

On 18/09/13 16:12, JD Wong wrote:

> Thanks for the helpful explanation!  So it looks like the best option is
> to create a decoy project.xml containing only my desired source, then
> running the project_build script on it, unless there is some other way I
> haven't thought of.
>
> Cheers,
> - JD
>
>
> On Wed, Sep 18, 2013 at 4:47 AM, Julie Sullivan <[hidden email]
> <mailto:[hidden email]>> wrote:
>
>     Yes, that's it! Unfortunately, ant isolates each individual task in
>     its own ClassLoader, which prevents the different tasks from sharing
>     database connections. Each ClassLoader will end up having its own
>     connection pool, with a maximum number of connections as set in the
>     properties. Moreover, it doesn't tend to allow those ClassLoaders or
>     database connection pools to be garbage collected once the task has
>     finished. Therefore, each individual task will leave several idle
>     database connections lying around. We tried to work around this
>     problem a while back too, but were unsuccessful. Ant simply isolates
>     the individual tasks too well.
>
>
>     On 17/09/13 23:10, JD Wong wrote:
>
>         I called each source directly through the Dsource parameter
>         because I
>         only wanted to run the new sources.  Hopefully this is the cause
>         of the
>         problem!
>
>         Thanks,
>         -JD
>
>
>         On Tue, Sep 17, 2013 at 6:34 AM, Julie Sullivan
>         <[hidden email] <mailto:[hidden email]>
>         <mailto:[hidden email] <mailto:[hidden email]>>> wrote:
>
>              JD
>
>              Can you verify that you are using the project_build script? In
>              FlyMine we have 38 FASTA sources and don't have any problems:
>
>         https://github.com/intermine/____intermine/blob/dev/flymine/____project.xml#L173
>         <https://github.com/intermine/__intermine/blob/dev/flymine/__project.xml#L173>
>
>
>         <https://github.com/intermine/__intermine/blob/dev/flymine/__project.xml#L173
>         <https://github.com/intermine/intermine/blob/dev/flymine/project.xml#L173>>
>
>              So your build should be fine!
>
>              There is a maxConnections parameter for each one in either
>
>                       default.intermine.integrate.____properties
>                       default.intermine.webapp.____properties
>
>
>              You can increase the value and see if it helps?
>
>              If it doesn't it's probably the postgres installation
>         itself running out
>              of connections, in which case you could check whether you
>         have lots
>              of webapps deployed.  There are also postgres settings you can
>              change for altering the number of available connections, I
>         can point
>              these out if you're still having problems.
>
>              Julie
>
>
>              On 17/09/13 00:05, JD Wong wrote:
>
>                  Hi all,
>
>                  It's been a while since the last help request, and so
>         it's time
>                  to break
>                  that trend.
>
>                  I've recently expanded my build to include the genomic and
>                  protein fasta
>                  sources for all the species in wormbase along the lines
>         of 48 fasta
>                  based sources .
>
>                  Loading them, I get this new stack trace.
>
>                  It seems that the mine might be opening a new database
>                  connection for
>                  each source.  Can anybody comment on the root cause &
>         how to fix it?
>
>                  Thanks!
>
>                  BUILD FAILED
>
>         /nfs/wormbase/wormmine/____website-intermine-dev/acedb-____dev/intermine/imbuild/____integrate.xml:54:
>
>                  The following error occurred while executing this line:
>
>         /nfs/wormbase/wormmine/____website-intermine-dev/acedb-____dev/intermine/bio/sources/____fasta/build.xml:46:
>                  org.intermine.objectstore.____ObjectStoreException:
>         Failed to
>                  instantiate
>                  IntegrationWriter class:
>
>         org.intermine.dataloader.____IntegrationWriterDataTrackingI____mpl
>                            at
>
>         org.intermine.dataloader.____IntegrationWriterFactory.____getIntegrationWriter(____IntegrationWriterFactory.java:____75)
>                            at
>
>         org.intermine.task.____DirectDataLoaderTask.____getIntegrationWriter(____DirectDataLoaderTask.java:85)
>                            at
>
>         org.intermine.task.____DirectDataLoaderTask.execute(____DirectDataLoaderTask.java:__129)
>                            at
>
>         org.intermine.bio.____dataconversion.____FastaLoaderTask.execute(____FastaLoaderTask.java:200)
>                            at
>
>         org.apache.tools.ant.____UnknownElement.execute(____UnknownElement.java:291)
>                            at
>
>         sun.reflect.____GeneratedMethodAccessor360.____invoke(Unknown
>         Source)
>                            at
>
>         sun.reflect.____DelegatingMethodAccessorImpl.____invoke(____DelegatingMethodAccessorImpl.____java:25)
>                            at
>         java.lang.reflect.Method.____invoke(Method.java:597)
>                            at
>
>         org.apache.tools.ant.dispatch.____DispatchUtils.execute(____DispatchUtils.java:106)
>                            at
>         org.apache.tools.ant.Task.____perform(Task.java:348)
>                            at
>         org.apache.tools.ant.Target.____execute(Target.java:390)
>                            at
>
>         org.apache.tools.ant.Target.____performTasks(Target.java:411)
>                            at
>
>         org.apache.tools.ant.Project.____executeSortedTargets(Project.____java:1360)
>                            at
>
>         org.apache.tools.ant.helper.____SingleCheckExecutor.____executeTargets(____SingleCheckExecutor.java:38)
>                            at
>
>         org.apache.tools.ant.Project.____executeTargets(Project.java:____1212)
>                            at
>                  org.apache.tools.ant.taskdefs.____Ant.execute(Ant.java:441)
>                            at
>
>         org.intermine.task.Integrate.____performAction(Integrate.java:____223)
>                            at
>
>         org.intermine.task.Integrate.____performAction(Integrate.java:____136)
>                            at
>
>         org.intermine.task.Integrate.____execute(Integrate.java:127)
>                            at
>
>         org.apache.tools.ant.____UnknownElement.execute(____UnknownElement.java:291)
>                            at
>
>         sun.reflect.____GeneratedMethodAccessor4.____invoke(Unknown Source)
>                            at
>
>         sun.reflect.____DelegatingMethodAccessorImpl.____invoke(____DelegatingMethodAccessorImpl.____java:25)
>                            at
>         java.lang.reflect.Method.____invoke(Method.java:597)
>                            at
>
>         org.apache.tools.ant.dispatch.____DispatchUtils.execute(____DispatchUtils.java:106)
>                            at
>         org.apache.tools.ant.Task.____perform(Task.java:348)
>                            at
>         org.apache.tools.ant.Target.____execute(Target.java:390)
>                            at
>
>         org.apache.tools.ant.Target.____performTasks(Target.java:411)
>                            at
>
>         org.apache.tools.ant.Project.____executeSortedTargets(Project.____java:1360)
>                            at
>
>         org.apache.tools.ant.Project.____executeTarget(Project.java:____1329)
>                            at
>
>         org.apache.tools.ant.helper.____DefaultExecutor.____executeTargets(____DefaultExecutor.java:41)
>                            at
>
>         org.apache.tools.ant.Project.____executeTargets(Project.java:____1212)
>                            at
>         org.apache.tools.ant.Main.____runBuild(Main.java:801)
>                            at
>         org.apache.tools.ant.Main.____startAnt(Main.java:218)
>                            at
>
>         org.apache.tools.ant.launch.____Launcher.run(Launcher.java:____280)
>                            at
>
>         org.apache.tools.ant.launch.____Launcher.main(Launcher.java:____109)
>                  Caused by: java.lang.reflect.____InvocationTargetException
>                            at
>
>         sun.reflect.____NativeMethodAccessorImpl.____invoke0(Native Method)
>                            at
>
>         sun.reflect.____NativeMethodAccessorImpl.____invoke(____NativeMethodAccessorImpl.java:____39)
>                            at
>
>         sun.reflect.____DelegatingMethodAccessorImpl.____invoke(____DelegatingMethodAccessorImpl.____java:25)
>                            at
>         java.lang.reflect.Method.____invoke(Method.java:597)
>                            at
>
>         org.intermine.dataloader.____IntegrationWriterFactory.____getIntegrationWriter(____IntegrationWriterFactory.java:____65)
>                            ... 34 more
>                  Caused by:
>         org.intermine.objectstore.____ObjectStoreException:
>
>                  Failed to
>                  instantiate ObjectStoreWriter class:
>
>         org.intermine.objectstore.____intermine.____ObjectStoreWriterInterMineImpl____,
>
>                  osAlias: os.production - java.lang.RuntimeException:
>         can't open
>                  datasource for {platform=PostgreSQL,
>                  datasource.dataSourceName=db.____production,
>                  datasource.maxConnections=50,
>                  datasource.databaseName=____wormmine-test-t,
>
>         datasource.class=org.____postgresql.ds.____PGPoolingDataSource,
>                  datasource.password=#####, datasource.user=intermine,
>                  driver=org.postgresql.Driver,
>         datasource.serverName=____localhost}
>                  org.postgresql.Driver PostgreSQL
>                            at
>
>         org.intermine.objectstore.____ObjectStoreWriterFactory.____getObjectStoreWriter(____ObjectStoreWriterFactory.java:____89)
>                            at
>
>         org.intermine.dataloader.____IntegrationWriterDataTrackingI____mpl.getInstance(____IntegrationWriterDataTrackingI____mpl.java:120)
>                            at
>
>         org.intermine.dataloader.____IntegrationWriterDataTrackingI____mpl.getInstance(____IntegrationWriterDataTrackingI____mpl.java:80)
>
>                            ... 39 more
>                  Caused by: java.lang.RuntimeException: can't open
>         datasource for
>                  {platform=PostgreSQL,
>         datasource.dataSourceName=db.____production,
>                  datasource.maxConnections=50,
>                  datasource.databaseName=____wormmine-test-t,
>
>         datasource.class=org.____postgresql.ds.____PGPoolingDataSource,
>                  datasource.password=#####, datasource.user=intermine,
>                  driver=org.postgresql.Driver,
>         datasource.serverName=____localhost}
>                  org.postgresql.Driver PostgreSQL
>                            at
>
>         org.intermine.sql.Database.____getConnection(Database.java:____106)
>                            at
>
>         org.intermine.objectstore.____intermine.____ObjectStoreInterMineImpl.____getConnection(____ObjectStoreInterMineImpl.java:____256)
>                            at
>
>         org.intermine.objectstore.____intermine.____ObjectStoreWriterInterMineImpl____.<init>(____ObjectStoreWriterInterMineImpl____.java:117)
>                            at
>
>         sun.reflect.____NativeConstructorAccessorImpl.____newInstance0(Native
>         Method)
>                            at
>
>         sun.reflect.____NativeConstructorAccessorImpl.____newInstance(____NativeConstructorAccessorImpl.____java:39)
>                            at
>
>         sun.reflect.____DelegatingConstructorAccessorI____mpl.newInstance(____DelegatingConstructorAccessorI____mpl.java:27)
>                            at
>
>         java.lang.reflect.Constructor.____newInstance(Constructor.__java:__513)
>                            at
>
>         org.intermine.objectstore.____ObjectStoreWriterFactory.____getObjectStoreWriter(____ObjectStoreWriterFactory.java:____78)
>                            ... 41 more
>                  Caused by: org.postgresql.util.____PSQLException: FATAL:
>
>                  connection limit
>                  exceeded for non-superusers
>                            at
>
>         org.postgresql.core.v3.____ConnectionFactoryImpl.____readStartupMessages(____ConnectionFactoryImpl.java:____464)
>                            at
>
>         org.postgresql.core.v3.____ConnectionFactoryImpl.____openConnectionImpl(____ConnectionFactoryImpl.java:____112)
>                            at
>
>         org.postgresql.core.____ConnectionFactory.____openConnection(____ConnectionFactory.java:66)
>                            at
>
>         org.postgresql.jdbc2.____AbstractJdbc2Connection.<init>____(AbstractJdbc2Connection.__java:__125)
>                            at
>
>         org.postgresql.jdbc3.____AbstractJdbc3Connection.<init>____(AbstractJdbc3Connection.__java:__30)
>                            at
>
>         org.postgresql.jdbc3g.____AbstractJdbc3gConnection.<____init>(____AbstractJdbc3gConnection.java:____22)
>                            at
>
>         org.postgresql.jdbc3g.____Jdbc3gConnection.<init>(____Jdbc3gConnection.java:24)
>                            at
>
>         org.postgresql.Driver.____makeConnection(Driver.java:____393)
>                            at
>         org.postgresql.Driver.connect(____Driver.java:267)
>                            at
>
>         java.sql.DriverManager.____getConnection(DriverManager.____java:582)
>                            at
>
>         java.sql.DriverManager.____getConnection(DriverManager.____java:185)
>                            at
>
>         org.postgresql.ds.common.____BaseDataSource.getConnection(____BaseDataSource.java:91)
>                            at
>
>         org.postgresql.ds.common.____BaseDataSource.getConnection(____BaseDataSource.java:74)
>                            at
>
>         org.postgresql.ds.____PGConnectionPoolDataSource.____getPooledConnection(____PGConnectionPoolDataSource.____java:60)
>                            at
>
>         org.postgresql.ds.jdbc23.____AbstractJdbc23PoolingDataSourc____e.getPooledConnection(____AbstractJdbc23PoolingDataSourc____e.java:429)
>                            at
>
>         org.postgresql.ds.jdbc23.____AbstractJdbc23PoolingDataSourc____e.getConnection(____AbstractJdbc23PoolingDataSourc____e.java:356)
>                            at
>
>         org.intermine.sql.Database.____getConnection(Database.java:____104)
>                            ... 48 more
>
>
>
>                  ___________________________________________________
>                  dev mailing list
>         [hidden email] <mailto:[hidden email]>
>         <mailto:[hidden email] <mailto:[hidden email]>>
>         http://mail.intermine.org/cgi-____bin/mailman/listinfo/dev
>         <http://mail.intermine.org/cgi-__bin/mailman/listinfo/dev>
>
>         <http://mail.intermine.org/__cgi-bin/mailman/listinfo/dev
>         <http://mail.intermine.org/cgi-bin/mailman/listinfo/dev>>
>
>
>

_______________________________________________
dev mailing list
[hidden email]
http://mail.intermine.org/cgi-bin/mailman/listinfo/dev
Reply | Threaded
Open this post in threaded view
|

Re: Overloading postgres on build

Sierra Moxon
Yes! :)

On Sep 18, 2013, at 9:26 AM, Julie Sullivan <[hidden email]> wrote:

> Yes, I suppose that's the best way, although that doesn't seem great!
>
> Would it be useful to be able to pass the source names you want to run to the project_build script? Or maybe be able to specify different project XML files?
>
> On 18/09/13 16:12, JD Wong wrote:
>> Thanks for the helpful explanation!  So it looks like the best option is
>> to create a decoy project.xml containing only my desired source, then
>> running the project_build script on it, unless there is some other way I
>> haven't thought of.
>>
>> Cheers,
>> - JD
>>
>>
>> On Wed, Sep 18, 2013 at 4:47 AM, Julie Sullivan <[hidden email]
>> <mailto:[hidden email]>> wrote:
>>
>>    Yes, that's it! Unfortunately, ant isolates each individual task in
>>    its own ClassLoader, which prevents the different tasks from sharing
>>    database connections. Each ClassLoader will end up having its own
>>    connection pool, with a maximum number of connections as set in the
>>    properties. Moreover, it doesn't tend to allow those ClassLoaders or
>>    database connection pools to be garbage collected once the task has
>>    finished. Therefore, each individual task will leave several idle
>>    database connections lying around. We tried to work around this
>>    problem a while back too, but were unsuccessful. Ant simply isolates
>>    the individual tasks too well.
>>
>>
>>    On 17/09/13 23:10, JD Wong wrote:
>>
>>        I called each source directly through the Dsource parameter
>>        because I
>>        only wanted to run the new sources.  Hopefully this is the cause
>>        of the
>>        problem!
>>
>>        Thanks,
>>        -JD
>>
>>
>>        On Tue, Sep 17, 2013 at 6:34 AM, Julie Sullivan
>>        <[hidden email] <mailto:[hidden email]>
>>        <mailto:[hidden email] <mailto:[hidden email]>>> wrote:
>>
>>             JD
>>
>>             Can you verify that you are using the project_build script? In
>>             FlyMine we have 38 FASTA sources and don't have any problems:
>>
>>        https://github.com/intermine/____intermine/blob/dev/flymine/____project.xml#L173
>>        <https://github.com/intermine/__intermine/blob/dev/flymine/__project.xml#L173>
>>
>>
>>        <https://github.com/intermine/__intermine/blob/dev/flymine/__project.xml#L173
>>        <https://github.com/intermine/intermine/blob/dev/flymine/project.xml#L173>>
>>
>>             So your build should be fine!
>>
>>             There is a maxConnections parameter for each one in either
>>
>>                      default.intermine.integrate.____properties
>>                      default.intermine.webapp.____properties
>>
>>
>>             You can increase the value and see if it helps?
>>
>>             If it doesn't it's probably the postgres installation
>>        itself running out
>>             of connections, in which case you could check whether you
>>        have lots
>>             of webapps deployed.  There are also postgres settings you can
>>             change for altering the number of available connections, I
>>        can point
>>             these out if you're still having problems.
>>
>>             Julie
>>
>>
>>             On 17/09/13 00:05, JD Wong wrote:
>>
>>                 Hi all,
>>
>>                 It's been a while since the last help request, and so
>>        it's time
>>                 to break
>>                 that trend.
>>
>>                 I've recently expanded my build to include the genomic and
>>                 protein fasta
>>                 sources for all the species in wormbase along the lines
>>        of 48 fasta
>>                 based sources .
>>
>>                 Loading them, I get this new stack trace.
>>
>>                 It seems that the mine might be opening a new database
>>                 connection for
>>                 each source.  Can anybody comment on the root cause &
>>        how to fix it?
>>
>>                 Thanks!
>>
>>                 BUILD FAILED
>>
>>        /nfs/wormbase/wormmine/____website-intermine-dev/acedb-____dev/intermine/imbuild/____integrate.xml:54:
>>
>>                 The following error occurred while executing this line:
>>
>>        /nfs/wormbase/wormmine/____website-intermine-dev/acedb-____dev/intermine/bio/sources/____fasta/build.xml:46:
>>                 org.intermine.objectstore.____ObjectStoreException:
>>        Failed to
>>                 instantiate
>>                 IntegrationWriter class:
>>
>>        org.intermine.dataloader.____IntegrationWriterDataTrackingI____mpl
>>                           at
>>
>>        org.intermine.dataloader.____IntegrationWriterFactory.____getIntegrationWriter(____IntegrationWriterFactory.java:____75)
>>                           at
>>
>>        org.intermine.task.____DirectDataLoaderTask.____getIntegrationWriter(____DirectDataLoaderTask.java:85)
>>                           at
>>
>>        org.intermine.task.____DirectDataLoaderTask.execute(____DirectDataLoaderTask.java:__129)
>>                           at
>>
>>        org.intermine.bio.____dataconversion.____FastaLoaderTask.execute(____FastaLoaderTask.java:200)
>>                           at
>>
>>        org.apache.tools.ant.____UnknownElement.execute(____UnknownElement.java:291)
>>                           at
>>
>>        sun.reflect.____GeneratedMethodAccessor360.____invoke(Unknown
>>        Source)
>>                           at
>>
>>        sun.reflect.____DelegatingMethodAccessorImpl.____invoke(____DelegatingMethodAccessorImpl.____java:25)
>>                           at
>>        java.lang.reflect.Method.____invoke(Method.java:597)
>>                           at
>>
>>        org.apache.tools.ant.dispatch.____DispatchUtils.execute(____DispatchUtils.java:106)
>>                           at
>>        org.apache.tools.ant.Task.____perform(Task.java:348)
>>                           at
>>        org.apache.tools.ant.Target.____execute(Target.java:390)
>>                           at
>>
>>        org.apache.tools.ant.Target.____performTasks(Target.java:411)
>>                           at
>>
>>        org.apache.tools.ant.Project.____executeSortedTargets(Project.____java:1360)
>>                           at
>>
>>        org.apache.tools.ant.helper.____SingleCheckExecutor.____executeTargets(____SingleCheckExecutor.java:38)
>>                           at
>>
>>        org.apache.tools.ant.Project.____executeTargets(Project.java:____1212)
>>                           at
>>                 org.apache.tools.ant.taskdefs.____Ant.execute(Ant.java:441)
>>                           at
>>
>>        org.intermine.task.Integrate.____performAction(Integrate.java:____223)
>>                           at
>>
>>        org.intermine.task.Integrate.____performAction(Integrate.java:____136)
>>                           at
>>
>>        org.intermine.task.Integrate.____execute(Integrate.java:127)
>>                           at
>>
>>        org.apache.tools.ant.____UnknownElement.execute(____UnknownElement.java:291)
>>                           at
>>
>>        sun.reflect.____GeneratedMethodAccessor4.____invoke(Unknown Source)
>>                           at
>>
>>        sun.reflect.____DelegatingMethodAccessorImpl.____invoke(____DelegatingMethodAccessorImpl.____java:25)
>>                           at
>>        java.lang.reflect.Method.____invoke(Method.java:597)
>>                           at
>>
>>        org.apache.tools.ant.dispatch.____DispatchUtils.execute(____DispatchUtils.java:106)
>>                           at
>>        org.apache.tools.ant.Task.____perform(Task.java:348)
>>                           at
>>        org.apache.tools.ant.Target.____execute(Target.java:390)
>>                           at
>>
>>        org.apache.tools.ant.Target.____performTasks(Target.java:411)
>>                           at
>>
>>        org.apache.tools.ant.Project.____executeSortedTargets(Project.____java:1360)
>>                           at
>>
>>        org.apache.tools.ant.Project.____executeTarget(Project.java:____1329)
>>                           at
>>
>>        org.apache.tools.ant.helper.____DefaultExecutor.____executeTargets(____DefaultExecutor.java:41)
>>                           at
>>
>>        org.apache.tools.ant.Project.____executeTargets(Project.java:____1212)
>>                           at
>>        org.apache.tools.ant.Main.____runBuild(Main.java:801)
>>                           at
>>        org.apache.tools.ant.Main.____startAnt(Main.java:218)
>>                           at
>>
>>        org.apache.tools.ant.launch.____Launcher.run(Launcher.java:____280)
>>                           at
>>
>>        org.apache.tools.ant.launch.____Launcher.main(Launcher.java:____109)
>>                 Caused by: java.lang.reflect.____InvocationTargetException
>>                           at
>>
>>        sun.reflect.____NativeMethodAccessorImpl.____invoke0(Native Method)
>>                           at
>>
>>        sun.reflect.____NativeMethodAccessorImpl.____invoke(____NativeMethodAccessorImpl.java:____39)
>>                           at
>>
>>        sun.reflect.____DelegatingMethodAccessorImpl.____invoke(____DelegatingMethodAccessorImpl.____java:25)
>>                           at
>>        java.lang.reflect.Method.____invoke(Method.java:597)
>>                           at
>>
>>        org.intermine.dataloader.____IntegrationWriterFactory.____getIntegrationWriter(____IntegrationWriterFactory.java:____65)
>>                           ... 34 more
>>                 Caused by:
>>        org.intermine.objectstore.____ObjectStoreException:
>>
>>                 Failed to
>>                 instantiate ObjectStoreWriter class:
>>
>>        org.intermine.objectstore.____intermine.____ObjectStoreWriterInterMineImpl____,
>>
>>                 osAlias: os.production - java.lang.RuntimeException:
>>        can't open
>>                 datasource for {platform=PostgreSQL,
>>                 datasource.dataSourceName=db.____production,
>>                 datasource.maxConnections=50,
>>                 datasource.databaseName=____wormmine-test-t,
>>
>>        datasource.class=org.____postgresql.ds.____PGPoolingDataSource,
>>                 datasource.password=#####, datasource.user=intermine,
>>                 driver=org.postgresql.Driver,
>>        datasource.serverName=____localhost}
>>                 org.postgresql.Driver PostgreSQL
>>                           at
>>
>>        org.intermine.objectstore.____ObjectStoreWriterFactory.____getObjectStoreWriter(____ObjectStoreWriterFactory.java:____89)
>>                           at
>>
>>        org.intermine.dataloader.____IntegrationWriterDataTrackingI____mpl.getInstance(____IntegrationWriterDataTrackingI____mpl.java:120)
>>                           at
>>
>>        org.intermine.dataloader.____IntegrationWriterDataTrackingI____mpl.getInstance(____IntegrationWriterDataTrackingI____mpl.java:80)
>>
>>                           ... 39 more
>>                 Caused by: java.lang.RuntimeException: can't open
>>        datasource for
>>                 {platform=PostgreSQL,
>>        datasource.dataSourceName=db.____production,
>>                 datasource.maxConnections=50,
>>                 datasource.databaseName=____wormmine-test-t,
>>
>>        datasource.class=org.____postgresql.ds.____PGPoolingDataSource,
>>                 datasource.password=#####, datasource.user=intermine,
>>                 driver=org.postgresql.Driver,
>>        datasource.serverName=____localhost}
>>                 org.postgresql.Driver PostgreSQL
>>                           at
>>
>>        org.intermine.sql.Database.____getConnection(Database.java:____106)
>>                           at
>>
>>        org.intermine.objectstore.____intermine.____ObjectStoreInterMineImpl.____getConnection(____ObjectStoreInterMineImpl.java:____256)
>>                           at
>>
>>        org.intermine.objectstore.____intermine.____ObjectStoreWriterInterMineImpl____.<init>(____ObjectStoreWriterInterMineImpl____.java:117)
>>                           at
>>
>>        sun.reflect.____NativeConstructorAccessorImpl.____newInstance0(Native
>>        Method)
>>                           at
>>
>>        sun.reflect.____NativeConstructorAccessorImpl.____newInstance(____NativeConstructorAccessorImpl.____java:39)
>>                           at
>>
>>        sun.reflect.____DelegatingConstructorAccessorI____mpl.newInstance(____DelegatingConstructorAccessorI____mpl.java:27)
>>                           at
>>
>>        java.lang.reflect.Constructor.____newInstance(Constructor.__java:__513)
>>                           at
>>
>>        org.intermine.objectstore.____ObjectStoreWriterFactory.____getObjectStoreWriter(____ObjectStoreWriterFactory.java:____78)
>>                           ... 41 more
>>                 Caused by: org.postgresql.util.____PSQLException: FATAL:
>>
>>                 connection limit
>>                 exceeded for non-superusers
>>                           at
>>
>>        org.postgresql.core.v3.____ConnectionFactoryImpl.____readStartupMessages(____ConnectionFactoryImpl.java:____464)
>>                           at
>>
>>        org.postgresql.core.v3.____ConnectionFactoryImpl.____openConnectionImpl(____ConnectionFactoryImpl.java:____112)
>>                           at
>>
>>        org.postgresql.core.____ConnectionFactory.____openConnection(____ConnectionFactory.java:66)
>>                           at
>>
>>        org.postgresql.jdbc2.____AbstractJdbc2Connection.<init>____(AbstractJdbc2Connection.__java:__125)
>>                           at
>>
>>        org.postgresql.jdbc3.____AbstractJdbc3Connection.<init>____(AbstractJdbc3Connection.__java:__30)
>>                           at
>>
>>        org.postgresql.jdbc3g.____AbstractJdbc3gConnection.<____init>(____AbstractJdbc3gConnection.java:____22)
>>                           at
>>
>>        org.postgresql.jdbc3g.____Jdbc3gConnection.<init>(____Jdbc3gConnection.java:24)
>>                           at
>>
>>        org.postgresql.Driver.____makeConnection(Driver.java:____393)
>>                           at
>>        org.postgresql.Driver.connect(____Driver.java:267)
>>                           at
>>
>>        java.sql.DriverManager.____getConnection(DriverManager.____java:582)
>>                           at
>>
>>        java.sql.DriverManager.____getConnection(DriverManager.____java:185)
>>                           at
>>
>>        org.postgresql.ds.common.____BaseDataSource.getConnection(____BaseDataSource.java:91)
>>                           at
>>
>>        org.postgresql.ds.common.____BaseDataSource.getConnection(____BaseDataSource.java:74)
>>                           at
>>
>>        org.postgresql.ds.____PGConnectionPoolDataSource.____getPooledConnection(____PGConnectionPoolDataSource.____java:60)
>>                           at
>>
>>        org.postgresql.ds.jdbc23.____AbstractJdbc23PoolingDataSourc____e.getPooledConnection(____AbstractJdbc23PoolingDataSourc____e.java:429)
>>                           at
>>
>>        org.postgresql.ds.jdbc23.____AbstractJdbc23PoolingDataSourc____e.getConnection(____AbstractJdbc23PoolingDataSourc____e.java:356)
>>                           at
>>
>>        org.intermine.sql.Database.____getConnection(Database.java:____104)
>>                           ... 48 more
>>
>>
>>
>>                 ___________________________________________________
>>                 dev mailing list
>>        [hidden email] <mailto:[hidden email]>
>>        <mailto:[hidden email] <mailto:[hidden email]>>
>>        http://mail.intermine.org/cgi-____bin/mailman/listinfo/dev
>>        <http://mail.intermine.org/cgi-__bin/mailman/listinfo/dev>
>>
>>        <http://mail.intermine.org/__cgi-bin/mailman/listinfo/dev
>>        <http://mail.intermine.org/cgi-bin/mailman/listinfo/dev>>
>>
>>
>>
>
> _______________________________________________
> dev mailing list
> [hidden email]
> http://mail.intermine.org/cgi-bin/mailman/listinfo/dev


_______________________________________________
dev mailing list
[hidden email]
http://mail.intermine.org/cgi-bin/mailman/listinfo/dev
Reply | Threaded
Open this post in threaded view
|

Re: Overloading postgres on build

Jayaraman, Pushkala
In reply to this post by Julie Sullivan
This is what ive been doing to debug..
Just comment out other sources in my project.xml and keep the desired sources and run it!

-----Original Message-----
From: [hidden email] [mailto:[hidden email]] On Behalf Of Julie Sullivan
Sent: Wednesday, September 18, 2013 11:26 AM
To: JD Wong
Cc: [hidden email]
Subject: Re: [InterMine Dev] Overloading postgres on build

Yes, I suppose that's the best way, although that doesn't seem great!

Would it be useful to be able to pass the source names you want to run to the project_build script? Or maybe be able to specify different project XML files?

On 18/09/13 16:12, JD Wong wrote:

> Thanks for the helpful explanation!  So it looks like the best option
> is to create a decoy project.xml containing only my desired source,
> then running the project_build script on it, unless there is some
> other way I haven't thought of.
>
> Cheers,
> - JD
>
>
> On Wed, Sep 18, 2013 at 4:47 AM, Julie Sullivan <[hidden email]
> <mailto:[hidden email]>> wrote:
>
>     Yes, that's it! Unfortunately, ant isolates each individual task in
>     its own ClassLoader, which prevents the different tasks from sharing
>     database connections. Each ClassLoader will end up having its own
>     connection pool, with a maximum number of connections as set in the
>     properties. Moreover, it doesn't tend to allow those ClassLoaders or
>     database connection pools to be garbage collected once the task has
>     finished. Therefore, each individual task will leave several idle
>     database connections lying around. We tried to work around this
>     problem a while back too, but were unsuccessful. Ant simply isolates
>     the individual tasks too well.
>
>
>     On 17/09/13 23:10, JD Wong wrote:
>
>         I called each source directly through the Dsource parameter
>         because I
>         only wanted to run the new sources.  Hopefully this is the cause
>         of the
>         problem!
>
>         Thanks,
>         -JD
>
>
>         On Tue, Sep 17, 2013 at 6:34 AM, Julie Sullivan
>         <[hidden email] <mailto:[hidden email]>
>         <mailto:[hidden email] <mailto:[hidden email]>>> wrote:
>
>              JD
>
>              Can you verify that you are using the project_build script? In
>              FlyMine we have 38 FASTA sources and don't have any problems:
>
>         https://github.com/intermine/____intermine/blob/dev/flymine/____project.xml#L173
>        
> <https://github.com/intermine/__intermine/blob/dev/flymine/__project.x
> ml#L173>
>
>
>         <https://github.com/intermine/__intermine/blob/dev/flymine/__project.xml#L173
>        
> <https://github.com/intermine/intermine/blob/dev/flymine/project.xml#L
> 173>>
>
>              So your build should be fine!
>
>              There is a maxConnections parameter for each one in
> either
>
>                       default.intermine.integrate.____properties
>                       default.intermine.webapp.____properties
>
>
>              You can increase the value and see if it helps?
>
>              If it doesn't it's probably the postgres installation
>         itself running out
>              of connections, in which case you could check whether you
>         have lots
>              of webapps deployed.  There are also postgres settings you can
>              change for altering the number of available connections, I
>         can point
>              these out if you're still having problems.
>
>              Julie
>
>
>              On 17/09/13 00:05, JD Wong wrote:
>
>                  Hi all,
>
>                  It's been a while since the last help request, and so
>         it's time
>                  to break
>                  that trend.
>
>                  I've recently expanded my build to include the genomic and
>                  protein fasta
>                  sources for all the species in wormbase along the lines
>         of 48 fasta
>                  based sources .
>
>                  Loading them, I get this new stack trace.
>
>                  It seems that the mine might be opening a new database
>                  connection for
>                  each source.  Can anybody comment on the root cause &
>         how to fix it?
>
>                  Thanks!
>
>                  BUILD FAILED
>
>         /nfs/wormbase/wormmine/____website-intermine-dev/acedb-____dev/intermine/imbuild/____integrate.xml:54:
>
>                  The following error occurred while executing this line:
>
>         /nfs/wormbase/wormmine/____website-intermine-dev/acedb-____dev/intermine/bio/sources/____fasta/build.xml:46:
>                  org.intermine.objectstore.____ObjectStoreException:
>         Failed to
>                  instantiate
>                  IntegrationWriter class:
>
>         org.intermine.dataloader.____IntegrationWriterDataTrackingI____mpl
>                            at
>
>         org.intermine.dataloader.____IntegrationWriterFactory.____getIntegrationWriter(____IntegrationWriterFactory.java:____75)
>                            at
>
>         org.intermine.task.____DirectDataLoaderTask.____getIntegrationWriter(____DirectDataLoaderTask.java:85)
>                            at
>
>         org.intermine.task.____DirectDataLoaderTask.execute(____DirectDataLoaderTask.java:__129)
>                            at
>
>         org.intermine.bio.____dataconversion.____FastaLoaderTask.execute(____FastaLoaderTask.java:200)
>                            at
>
>         org.apache.tools.ant.____UnknownElement.execute(____UnknownElement.java:291)
>                            at
>
>         sun.reflect.____GeneratedMethodAccessor360.____invoke(Unknown
>         Source)
>                            at
>
>         sun.reflect.____DelegatingMethodAccessorImpl.____invoke(____DelegatingMethodAccessorImpl.____java:25)
>                            at
>         java.lang.reflect.Method.____invoke(Method.java:597)
>                            at
>
>         org.apache.tools.ant.dispatch.____DispatchUtils.execute(____DispatchUtils.java:106)
>                            at
>         org.apache.tools.ant.Task.____perform(Task.java:348)
>                            at
>         org.apache.tools.ant.Target.____execute(Target.java:390)
>                            at
>
>         org.apache.tools.ant.Target.____performTasks(Target.java:411)
>                            at
>
>         org.apache.tools.ant.Project.____executeSortedTargets(Project.____java:1360)
>                            at
>
>         org.apache.tools.ant.helper.____SingleCheckExecutor.____executeTargets(____SingleCheckExecutor.java:38)
>                            at
>
>         org.apache.tools.ant.Project.____executeTargets(Project.java:____1212)
>                            at
>                  org.apache.tools.ant.taskdefs.____Ant.execute(Ant.java:441)
>                            at
>
>         org.intermine.task.Integrate.____performAction(Integrate.java:____223)
>                            at
>
>         org.intermine.task.Integrate.____performAction(Integrate.java:____136)
>                            at
>
>         org.intermine.task.Integrate.____execute(Integrate.java:127)
>                            at
>
>         org.apache.tools.ant.____UnknownElement.execute(____UnknownElement.java:291)
>                            at
>
>         sun.reflect.____GeneratedMethodAccessor4.____invoke(Unknown Source)
>                            at
>
>         sun.reflect.____DelegatingMethodAccessorImpl.____invoke(____DelegatingMethodAccessorImpl.____java:25)
>                            at
>         java.lang.reflect.Method.____invoke(Method.java:597)
>                            at
>
>         org.apache.tools.ant.dispatch.____DispatchUtils.execute(____DispatchUtils.java:106)
>                            at
>         org.apache.tools.ant.Task.____perform(Task.java:348)
>                            at
>         org.apache.tools.ant.Target.____execute(Target.java:390)
>                            at
>
>         org.apache.tools.ant.Target.____performTasks(Target.java:411)
>                            at
>
>         org.apache.tools.ant.Project.____executeSortedTargets(Project.____java:1360)
>                            at
>
>         org.apache.tools.ant.Project.____executeTarget(Project.java:____1329)
>                            at
>
>         org.apache.tools.ant.helper.____DefaultExecutor.____executeTargets(____DefaultExecutor.java:41)
>                            at
>
>         org.apache.tools.ant.Project.____executeTargets(Project.java:____1212)
>                            at
>         org.apache.tools.ant.Main.____runBuild(Main.java:801)
>                            at
>         org.apache.tools.ant.Main.____startAnt(Main.java:218)
>                            at
>
>         org.apache.tools.ant.launch.____Launcher.run(Launcher.java:____280)
>                            at
>
>         org.apache.tools.ant.launch.____Launcher.main(Launcher.java:____109)
>                  Caused by: java.lang.reflect.____InvocationTargetException
>                            at
>
>         sun.reflect.____NativeMethodAccessorImpl.____invoke0(Native Method)
>                            at
>
>         sun.reflect.____NativeMethodAccessorImpl.____invoke(____NativeMethodAccessorImpl.java:____39)
>                            at
>
>         sun.reflect.____DelegatingMethodAccessorImpl.____invoke(____DelegatingMethodAccessorImpl.____java:25)
>                            at
>         java.lang.reflect.Method.____invoke(Method.java:597)
>                            at
>
>         org.intermine.dataloader.____IntegrationWriterFactory.____getIntegrationWriter(____IntegrationWriterFactory.java:____65)
>                            ... 34 more
>                  Caused by:
>         org.intermine.objectstore.____ObjectStoreException:
>
>                  Failed to
>                  instantiate ObjectStoreWriter class:
>
>        
> org.intermine.objectstore.____intermine.____ObjectStoreWriterInterMine
> Impl____,
>
>                  osAlias: os.production - java.lang.RuntimeException:
>         can't open
>                  datasource for {platform=PostgreSQL,
>                  datasource.dataSourceName=db.____production,
>                  datasource.maxConnections=50,
>                  datasource.databaseName=____wormmine-test-t,
>
>         datasource.class=org.____postgresql.ds.____PGPoolingDataSource,
>                  datasource.password=#####, datasource.user=intermine,
>                  driver=org.postgresql.Driver,
>         datasource.serverName=____localhost}
>                  org.postgresql.Driver PostgreSQL
>                            at
>
>         org.intermine.objectstore.____ObjectStoreWriterFactory.____getObjectStoreWriter(____ObjectStoreWriterFactory.java:____89)
>                            at
>
>         org.intermine.dataloader.____IntegrationWriterDataTrackingI____mpl.getInstance(____IntegrationWriterDataTrackingI____mpl.java:120)
>                            at
>
>        
> org.intermine.dataloader.____IntegrationWriterDataTrackingI____mpl.get
> Instance(____IntegrationWriterDataTrackingI____mpl.java:80)
>
>                            ... 39 more
>                  Caused by: java.lang.RuntimeException: can't open
>         datasource for
>                  {platform=PostgreSQL,
>         datasource.dataSourceName=db.____production,
>                  datasource.maxConnections=50,
>                  datasource.databaseName=____wormmine-test-t,
>
>         datasource.class=org.____postgresql.ds.____PGPoolingDataSource,
>                  datasource.password=#####, datasource.user=intermine,
>                  driver=org.postgresql.Driver,
>         datasource.serverName=____localhost}
>                  org.postgresql.Driver PostgreSQL
>                            at
>
>         org.intermine.sql.Database.____getConnection(Database.java:____106)
>                            at
>
>         org.intermine.objectstore.____intermine.____ObjectStoreInterMineImpl.____getConnection(____ObjectStoreInterMineImpl.java:____256)
>                            at
>
>         org.intermine.objectstore.____intermine.____ObjectStoreWriterInterMineImpl____.<init>(____ObjectStoreWriterInterMineImpl____.java:117)
>                            at
>
>         sun.reflect.____NativeConstructorAccessorImpl.____newInstance0(Native
>         Method)
>                            at
>
>         sun.reflect.____NativeConstructorAccessorImpl.____newInstance(____NativeConstructorAccessorImpl.____java:39)
>                            at
>
>         sun.reflect.____DelegatingConstructorAccessorI____mpl.newInstance(____DelegatingConstructorAccessorI____mpl.java:27)
>                            at
>
>         java.lang.reflect.Constructor.____newInstance(Constructor.__java:__513)
>                            at
>
>         org.intermine.objectstore.____ObjectStoreWriterFactory.____getObjectStoreWriter(____ObjectStoreWriterFactory.java:____78)
>                            ... 41 more
>                  Caused by: org.postgresql.util.____PSQLException: FATAL:
>
>                  connection limit
>                  exceeded for non-superusers
>                            at
>
>         org.postgresql.core.v3.____ConnectionFactoryImpl.____readStartupMessages(____ConnectionFactoryImpl.java:____464)
>                            at
>
>         org.postgresql.core.v3.____ConnectionFactoryImpl.____openConnectionImpl(____ConnectionFactoryImpl.java:____112)
>                            at
>
>         org.postgresql.core.____ConnectionFactory.____openConnection(____ConnectionFactory.java:66)
>                            at
>
>         org.postgresql.jdbc2.____AbstractJdbc2Connection.<init>____(AbstractJdbc2Connection.__java:__125)
>                            at
>
>         org.postgresql.jdbc3.____AbstractJdbc3Connection.<init>____(AbstractJdbc3Connection.__java:__30)
>                            at
>
>         org.postgresql.jdbc3g.____AbstractJdbc3gConnection.<____init>(____AbstractJdbc3gConnection.java:____22)
>                            at
>
>         org.postgresql.jdbc3g.____Jdbc3gConnection.<init>(____Jdbc3gConnection.java:24)
>                            at
>
>         org.postgresql.Driver.____makeConnection(Driver.java:____393)
>                            at
>         org.postgresql.Driver.connect(____Driver.java:267)
>                            at
>
>         java.sql.DriverManager.____getConnection(DriverManager.____java:582)
>                            at
>
>         java.sql.DriverManager.____getConnection(DriverManager.____java:185)
>                            at
>
>         org.postgresql.ds.common.____BaseDataSource.getConnection(____BaseDataSource.java:91)
>                            at
>
>         org.postgresql.ds.common.____BaseDataSource.getConnection(____BaseDataSource.java:74)
>                            at
>
>         org.postgresql.ds.____PGConnectionPoolDataSource.____getPooledConnection(____PGConnectionPoolDataSource.____java:60)
>                            at
>
>         org.postgresql.ds.jdbc23.____AbstractJdbc23PoolingDataSourc____e.getPooledConnection(____AbstractJdbc23PoolingDataSourc____e.java:429)
>                            at
>
>         org.postgresql.ds.jdbc23.____AbstractJdbc23PoolingDataSourc____e.getConnection(____AbstractJdbc23PoolingDataSourc____e.java:356)
>                            at
>
>         org.intermine.sql.Database.____getConnection(Database.java:____104)
>                            ... 48 more
>
>
>
>                  ___________________________________________________
>                  dev mailing list
>         [hidden email] <mailto:[hidden email]>
>         <mailto:[hidden email] <mailto:[hidden email]>>
>         http://mail.intermine.org/cgi-____bin/mailman/listinfo/dev
>         <http://mail.intermine.org/cgi-__bin/mailman/listinfo/dev>
>
>         <http://mail.intermine.org/__cgi-bin/mailman/listinfo/dev
>         <http://mail.intermine.org/cgi-bin/mailman/listinfo/dev>>
>
>
>

_______________________________________________
dev mailing list
[hidden email]
http://mail.intermine.org/cgi-bin/mailman/listinfo/dev

_______________________________________________
dev mailing list
[hidden email]
http://mail.intermine.org/cgi-bin/mailman/listinfo/dev
Reply | Threaded
Open this post in threaded view
|

Re: Overloading postgres on build

Julie Sullivan
In reply to this post by Sierra Moxon
I've made a ticket:

        https://github.com/intermine/intermine/issues/447

On 18/09/13 17:27, Sierra Moxon wrote:

> Yes! :)
>
> On Sep 18, 2013, at 9:26 AM, Julie Sullivan <[hidden email]> wrote:
>
>> Yes, I suppose that's the best way, although that doesn't seem great!
>>
>> Would it be useful to be able to pass the source names you want to run to the project_build script? Or maybe be able to specify different project XML files?
>>
>> On 18/09/13 16:12, JD Wong wrote:
>>> Thanks for the helpful explanation!  So it looks like the best option is
>>> to create a decoy project.xml containing only my desired source, then
>>> running the project_build script on it, unless there is some other way I
>>> haven't thought of.
>>>
>>> Cheers,
>>> - JD
>>>
>>>
>>> On Wed, Sep 18, 2013 at 4:47 AM, Julie Sullivan <[hidden email]
>>> <mailto:[hidden email]>> wrote:
>>>
>>>     Yes, that's it! Unfortunately, ant isolates each individual task in
>>>     its own ClassLoader, which prevents the different tasks from sharing
>>>     database connections. Each ClassLoader will end up having its own
>>>     connection pool, with a maximum number of connections as set in the
>>>     properties. Moreover, it doesn't tend to allow those ClassLoaders or
>>>     database connection pools to be garbage collected once the task has
>>>     finished. Therefore, each individual task will leave several idle
>>>     database connections lying around. We tried to work around this
>>>     problem a while back too, but were unsuccessful. Ant simply isolates
>>>     the individual tasks too well.
>>>
>>>
>>>     On 17/09/13 23:10, JD Wong wrote:
>>>
>>>         I called each source directly through the Dsource parameter
>>>         because I
>>>         only wanted to run the new sources.  Hopefully this is the cause
>>>         of the
>>>         problem!
>>>
>>>         Thanks,
>>>         -JD
>>>
>>>
>>>         On Tue, Sep 17, 2013 at 6:34 AM, Julie Sullivan
>>>         <[hidden email] <mailto:[hidden email]>
>>>         <mailto:[hidden email] <mailto:[hidden email]>>> wrote:
>>>
>>>              JD
>>>
>>>              Can you verify that you are using the project_build script? In
>>>              FlyMine we have 38 FASTA sources and don't have any problems:
>>>
>>>         https://github.com/intermine/____intermine/blob/dev/flymine/____project.xml#L173
>>>         <https://github.com/intermine/__intermine/blob/dev/flymine/__project.xml#L173>
>>>
>>>
>>>         <https://github.com/intermine/__intermine/blob/dev/flymine/__project.xml#L173
>>>         <https://github.com/intermine/intermine/blob/dev/flymine/project.xml#L173>>
>>>
>>>              So your build should be fine!
>>>
>>>              There is a maxConnections parameter for each one in either
>>>
>>>                       default.intermine.integrate.____properties
>>>                       default.intermine.webapp.____properties
>>>
>>>
>>>              You can increase the value and see if it helps?
>>>
>>>              If it doesn't it's probably the postgres installation
>>>         itself running out
>>>              of connections, in which case you could check whether you
>>>         have lots
>>>              of webapps deployed.  There are also postgres settings you can
>>>              change for altering the number of available connections, I
>>>         can point
>>>              these out if you're still having problems.
>>>
>>>              Julie
>>>
>>>
>>>              On 17/09/13 00:05, JD Wong wrote:
>>>
>>>                  Hi all,
>>>
>>>                  It's been a while since the last help request, and so
>>>         it's time
>>>                  to break
>>>                  that trend.
>>>
>>>                  I've recently expanded my build to include the genomic and
>>>                  protein fasta
>>>                  sources for all the species in wormbase along the lines
>>>         of 48 fasta
>>>                  based sources .
>>>
>>>                  Loading them, I get this new stack trace.
>>>
>>>                  It seems that the mine might be opening a new database
>>>                  connection for
>>>                  each source.  Can anybody comment on the root cause &
>>>         how to fix it?
>>>
>>>                  Thanks!
>>>
>>>                  BUILD FAILED
>>>
>>>         /nfs/wormbase/wormmine/____website-intermine-dev/acedb-____dev/intermine/imbuild/____integrate.xml:54:
>>>
>>>                  The following error occurred while executing this line:
>>>
>>>         /nfs/wormbase/wormmine/____website-intermine-dev/acedb-____dev/intermine/bio/sources/____fasta/build.xml:46:
>>>                  org.intermine.objectstore.____ObjectStoreException:
>>>         Failed to
>>>                  instantiate
>>>                  IntegrationWriter class:
>>>
>>>         org.intermine.dataloader.____IntegrationWriterDataTrackingI____mpl
>>>                            at
>>>
>>>         org.intermine.dataloader.____IntegrationWriterFactory.____getIntegrationWriter(____IntegrationWriterFactory.java:____75)
>>>                            at
>>>
>>>         org.intermine.task.____DirectDataLoaderTask.____getIntegrationWriter(____DirectDataLoaderTask.java:85)
>>>                            at
>>>
>>>         org.intermine.task.____DirectDataLoaderTask.execute(____DirectDataLoaderTask.java:__129)
>>>                            at
>>>
>>>         org.intermine.bio.____dataconversion.____FastaLoaderTask.execute(____FastaLoaderTask.java:200)
>>>                            at
>>>
>>>         org.apache.tools.ant.____UnknownElement.execute(____UnknownElement.java:291)
>>>                            at
>>>
>>>         sun.reflect.____GeneratedMethodAccessor360.____invoke(Unknown
>>>         Source)
>>>                            at
>>>
>>>         sun.reflect.____DelegatingMethodAccessorImpl.____invoke(____DelegatingMethodAccessorImpl.____java:25)
>>>                            at
>>>         java.lang.reflect.Method.____invoke(Method.java:597)
>>>                            at
>>>
>>>         org.apache.tools.ant.dispatch.____DispatchUtils.execute(____DispatchUtils.java:106)
>>>                            at
>>>         org.apache.tools.ant.Task.____perform(Task.java:348)
>>>                            at
>>>         org.apache.tools.ant.Target.____execute(Target.java:390)
>>>                            at
>>>
>>>         org.apache.tools.ant.Target.____performTasks(Target.java:411)
>>>                            at
>>>
>>>         org.apache.tools.ant.Project.____executeSortedTargets(Project.____java:1360)
>>>                            at
>>>
>>>         org.apache.tools.ant.helper.____SingleCheckExecutor.____executeTargets(____SingleCheckExecutor.java:38)
>>>                            at
>>>
>>>         org.apache.tools.ant.Project.____executeTargets(Project.java:____1212)
>>>                            at
>>>                  org.apache.tools.ant.taskdefs.____Ant.execute(Ant.java:441)
>>>                            at
>>>
>>>         org.intermine.task.Integrate.____performAction(Integrate.java:____223)
>>>                            at
>>>
>>>         org.intermine.task.Integrate.____performAction(Integrate.java:____136)
>>>                            at
>>>
>>>         org.intermine.task.Integrate.____execute(Integrate.java:127)
>>>                            at
>>>
>>>         org.apache.tools.ant.____UnknownElement.execute(____UnknownElement.java:291)
>>>                            at
>>>
>>>         sun.reflect.____GeneratedMethodAccessor4.____invoke(Unknown Source)
>>>                            at
>>>
>>>         sun.reflect.____DelegatingMethodAccessorImpl.____invoke(____DelegatingMethodAccessorImpl.____java:25)
>>>                            at
>>>         java.lang.reflect.Method.____invoke(Method.java:597)
>>>                            at
>>>
>>>         org.apache.tools.ant.dispatch.____DispatchUtils.execute(____DispatchUtils.java:106)
>>>                            at
>>>         org.apache.tools.ant.Task.____perform(Task.java:348)
>>>                            at
>>>         org.apache.tools.ant.Target.____execute(Target.java:390)
>>>                            at
>>>
>>>         org.apache.tools.ant.Target.____performTasks(Target.java:411)
>>>                            at
>>>
>>>         org.apache.tools.ant.Project.____executeSortedTargets(Project.____java:1360)
>>>                            at
>>>
>>>         org.apache.tools.ant.Project.____executeTarget(Project.java:____1329)
>>>                            at
>>>
>>>         org.apache.tools.ant.helper.____DefaultExecutor.____executeTargets(____DefaultExecutor.java:41)
>>>                            at
>>>
>>>         org.apache.tools.ant.Project.____executeTargets(Project.java:____1212)
>>>                            at
>>>         org.apache.tools.ant.Main.____runBuild(Main.java:801)
>>>                            at
>>>         org.apache.tools.ant.Main.____startAnt(Main.java:218)
>>>                            at
>>>
>>>         org.apache.tools.ant.launch.____Launcher.run(Launcher.java:____280)
>>>                            at
>>>
>>>         org.apache.tools.ant.launch.____Launcher.main(Launcher.java:____109)
>>>                  Caused by: java.lang.reflect.____InvocationTargetException
>>>                            at
>>>
>>>         sun.reflect.____NativeMethodAccessorImpl.____invoke0(Native Method)
>>>                            at
>>>
>>>         sun.reflect.____NativeMethodAccessorImpl.____invoke(____NativeMethodAccessorImpl.java:____39)
>>>                            at
>>>
>>>         sun.reflect.____DelegatingMethodAccessorImpl.____invoke(____DelegatingMethodAccessorImpl.____java:25)
>>>                            at
>>>         java.lang.reflect.Method.____invoke(Method.java:597)
>>>                            at
>>>
>>>         org.intermine.dataloader.____IntegrationWriterFactory.____getIntegrationWriter(____IntegrationWriterFactory.java:____65)
>>>                            ... 34 more
>>>                  Caused by:
>>>         org.intermine.objectstore.____ObjectStoreException:
>>>
>>>                  Failed to
>>>                  instantiate ObjectStoreWriter class:
>>>
>>>         org.intermine.objectstore.____intermine.____ObjectStoreWriterInterMineImpl____,
>>>
>>>                  osAlias: os.production - java.lang.RuntimeException:
>>>         can't open
>>>                  datasource for {platform=PostgreSQL,
>>>                  datasource.dataSourceName=db.____production,
>>>                  datasource.maxConnections=50,
>>>                  datasource.databaseName=____wormmine-test-t,
>>>
>>>         datasource.class=org.____postgresql.ds.____PGPoolingDataSource,
>>>                  datasource.password=#####, datasource.user=intermine,
>>>                  driver=org.postgresql.Driver,
>>>         datasource.serverName=____localhost}
>>>                  org.postgresql.Driver PostgreSQL
>>>                            at
>>>
>>>         org.intermine.objectstore.____ObjectStoreWriterFactory.____getObjectStoreWriter(____ObjectStoreWriterFactory.java:____89)
>>>                            at
>>>
>>>         org.intermine.dataloader.____IntegrationWriterDataTrackingI____mpl.getInstance(____IntegrationWriterDataTrackingI____mpl.java:120)
>>>                            at
>>>
>>>         org.intermine.dataloader.____IntegrationWriterDataTrackingI____mpl.getInstance(____IntegrationWriterDataTrackingI____mpl.java:80)
>>>
>>>                            ... 39 more
>>>                  Caused by: java.lang.RuntimeException: can't open
>>>         datasource for
>>>                  {platform=PostgreSQL,
>>>         datasource.dataSourceName=db.____production,
>>>                  datasource.maxConnections=50,
>>>                  datasource.databaseName=____wormmine-test-t,
>>>
>>>         datasource.class=org.____postgresql.ds.____PGPoolingDataSource,
>>>                  datasource.password=#####, datasource.user=intermine,
>>>                  driver=org.postgresql.Driver,
>>>         datasource.serverName=____localhost}
>>>                  org.postgresql.Driver PostgreSQL
>>>                            at
>>>
>>>         org.intermine.sql.Database.____getConnection(Database.java:____106)
>>>                            at
>>>
>>>         org.intermine.objectstore.____intermine.____ObjectStoreInterMineImpl.____getConnection(____ObjectStoreInterMineImpl.java:____256)
>>>                            at
>>>
>>>         org.intermine.objectstore.____intermine.____ObjectStoreWriterInterMineImpl____.<init>(____ObjectStoreWriterInterMineImpl____.java:117)
>>>                            at
>>>
>>>         sun.reflect.____NativeConstructorAccessorImpl.____newInstance0(Native
>>>         Method)
>>>                            at
>>>
>>>         sun.reflect.____NativeConstructorAccessorImpl.____newInstance(____NativeConstructorAccessorImpl.____java:39)
>>>                            at
>>>
>>>         sun.reflect.____DelegatingConstructorAccessorI____mpl.newInstance(____DelegatingConstructorAccessorI____mpl.java:27)
>>>                            at
>>>
>>>         java.lang.reflect.Constructor.____newInstance(Constructor.__java:__513)
>>>                            at
>>>
>>>         org.intermine.objectstore.____ObjectStoreWriterFactory.____getObjectStoreWriter(____ObjectStoreWriterFactory.java:____78)
>>>                            ... 41 more
>>>                  Caused by: org.postgresql.util.____PSQLException: FATAL:
>>>
>>>                  connection limit
>>>                  exceeded for non-superusers
>>>                            at
>>>
>>>         org.postgresql.core.v3.____ConnectionFactoryImpl.____readStartupMessages(____ConnectionFactoryImpl.java:____464)
>>>                            at
>>>
>>>         org.postgresql.core.v3.____ConnectionFactoryImpl.____openConnectionImpl(____ConnectionFactoryImpl.java:____112)
>>>                            at
>>>
>>>         org.postgresql.core.____ConnectionFactory.____openConnection(____ConnectionFactory.java:66)
>>>                            at
>>>
>>>         org.postgresql.jdbc2.____AbstractJdbc2Connection.<init>____(AbstractJdbc2Connection.__java:__125)
>>>                            at
>>>
>>>         org.postgresql.jdbc3.____AbstractJdbc3Connection.<init>____(AbstractJdbc3Connection.__java:__30)
>>>                            at
>>>
>>>         org.postgresql.jdbc3g.____AbstractJdbc3gConnection.<____init>(____AbstractJdbc3gConnection.java:____22)
>>>                            at
>>>
>>>         org.postgresql.jdbc3g.____Jdbc3gConnection.<init>(____Jdbc3gConnection.java:24)
>>>                            at
>>>
>>>         org.postgresql.Driver.____makeConnection(Driver.java:____393)
>>>                            at
>>>         org.postgresql.Driver.connect(____Driver.java:267)
>>>                            at
>>>
>>>         java.sql.DriverManager.____getConnection(DriverManager.____java:582)
>>>                            at
>>>
>>>         java.sql.DriverManager.____getConnection(DriverManager.____java:185)
>>>                            at
>>>
>>>         org.postgresql.ds.common.____BaseDataSource.getConnection(____BaseDataSource.java:91)
>>>                            at
>>>
>>>         org.postgresql.ds.common.____BaseDataSource.getConnection(____BaseDataSource.java:74)
>>>                            at
>>>
>>>         org.postgresql.ds.____PGConnectionPoolDataSource.____getPooledConnection(____PGConnectionPoolDataSource.____java:60)
>>>                            at
>>>
>>>         org.postgresql.ds.jdbc23.____AbstractJdbc23PoolingDataSourc____e.getPooledConnection(____AbstractJdbc23PoolingDataSourc____e.java:429)
>>>                            at
>>>
>>>         org.postgresql.ds.jdbc23.____AbstractJdbc23PoolingDataSourc____e.getConnection(____AbstractJdbc23PoolingDataSourc____e.java:356)
>>>                            at
>>>
>>>         org.intermine.sql.Database.____getConnection(Database.java:____104)
>>>                            ... 48 more
>>>
>>>
>>>
>>>                  ___________________________________________________
>>>                  dev mailing list
>>>         [hidden email] <mailto:[hidden email]>
>>>         <mailto:[hidden email] <mailto:[hidden email]>>
>>>         http://mail.intermine.org/cgi-____bin/mailman/listinfo/dev
>>>         <http://mail.intermine.org/cgi-__bin/mailman/listinfo/dev>
>>>
>>>         <http://mail.intermine.org/__cgi-bin/mailman/listinfo/dev
>>>         <http://mail.intermine.org/cgi-bin/mailman/listinfo/dev>>
>>>
>>>
>>>
>>
>> _______________________________________________
>> dev mailing list
>> [hidden email]
>> http://mail.intermine.org/cgi-bin/mailman/listinfo/dev
>
>

_______________________________________________
dev mailing list
[hidden email]
http://mail.intermine.org/cgi-bin/mailman/listinfo/dev
Reply | Threaded
Open this post in threaded view
|

Re: Overloading postgres on build

Julie Sullivan
In reply to this post by JD Wong
JD,

You can use the -a flag:

http://intermine.readthedocs.org/en/latest/database/database-building/build-script

Or see:

  ../bio/scripts/project_build -h


Note that it accepts the names of sources / postprocesses - and patterns
too, so for example in FlyMine I can run all fasta sources like so:

  ../bio/scripts/project_build -a *fasta* -v localhost /tmp

And I get this message:

running actions:
uniprot-fasta,flybase-dmel-gene-fasta,flybase-dpse-gene-fasta,flybase-dana-gene-fasta,flybase-dere-gene-fasta,flybase-dgri-gene-fasta,flybase-dmoj-gene-fasta,flybase-dper-gene-fasta,flybase-dsec-gene-fasta,flybase-dsim-gene-fasta,flybase-dvir-gene-fasta,flybase-dwil-gene-fasta,flybase-dyak-gene-fasta,flybase-dmel-cds-fasta,flybase-dpse-cds-fasta,flybase-dana-cds-fasta,flybase-dere-cds-fasta,flybase-dgri-cds-fasta,flybase-dmoj-cds-fasta,flybase-dper-cds-fasta,flybase-dsec-cds-fasta,flybase-dsim-cds-fasta,flybase-dvir-cds-fasta,flybase-dwil-cds-fasta,flybase-dyak-cds-fasta,flybase-dmel-5prime-utr-fasta,flybase-dmel-3prime-utr-fasta,flybase-dpse-5prime-utr-fasta,flybase-dpse-3prime-utr-fasta,flybase-dgri-5prime-utr-fasta,flybase-dper-5prime-utr-fasta,flybase-dsec-5prime-utr-fasta,flybase-dsim-5prime-utr-fasta,flybase-dyak-5prime-utr-fasta

Hopefully that's what you need!

Cheers
Julie

On 18/09/13 16:12, JD Wong wrote:

> Thanks for the helpful explanation!  So it looks like the best option is
> to create a decoy project.xml containing only my desired source, then
> running the project_build script on it, unless there is some other way I
> haven't thought of.
>
> Cheers,
> - JD
>
>
> On Wed, Sep 18, 2013 at 4:47 AM, Julie Sullivan <[hidden email]
> <mailto:[hidden email]>> wrote:
>
>     Yes, that's it! Unfortunately, ant isolates each individual task in
>     its own ClassLoader, which prevents the different tasks from sharing
>     database connections. Each ClassLoader will end up having its own
>     connection pool, with a maximum number of connections as set in the
>     properties. Moreover, it doesn't tend to allow those ClassLoaders or
>     database connection pools to be garbage collected once the task has
>     finished. Therefore, each individual task will leave several idle
>     database connections lying around. We tried to work around this
>     problem a while back too, but were unsuccessful. Ant simply isolates
>     the individual tasks too well.
>
>
>     On 17/09/13 23:10, JD Wong wrote:
>
>         I called each source directly through the Dsource parameter
>         because I
>         only wanted to run the new sources.  Hopefully this is the cause
>         of the
>         problem!
>
>         Thanks,
>         -JD
>
>
>         On Tue, Sep 17, 2013 at 6:34 AM, Julie Sullivan
>         <[hidden email] <mailto:[hidden email]>
>         <mailto:[hidden email] <mailto:[hidden email]>>> wrote:
>
>              JD
>
>              Can you verify that you are using the project_build script? In
>              FlyMine we have 38 FASTA sources and don't have any problems:
>
>         https://github.com/intermine/____intermine/blob/dev/flymine/____project.xml#L173
>         <https://github.com/intermine/__intermine/blob/dev/flymine/__project.xml#L173>
>
>
>         <https://github.com/intermine/__intermine/blob/dev/flymine/__project.xml#L173
>         <https://github.com/intermine/intermine/blob/dev/flymine/project.xml#L173>>
>
>              So your build should be fine!
>
>              There is a maxConnections parameter for each one in either
>
>                       default.intermine.integrate.____properties
>                       default.intermine.webapp.____properties
>
>
>              You can increase the value and see if it helps?
>
>              If it doesn't it's probably the postgres installation
>         itself running out
>              of connections, in which case you could check whether you
>         have lots
>              of webapps deployed.  There are also postgres settings you can
>              change for altering the number of available connections, I
>         can point
>              these out if you're still having problems.
>
>              Julie
>
>
>              On 17/09/13 00:05, JD Wong wrote:
>
>                  Hi all,
>
>                  It's been a while since the last help request, and so
>         it's time
>                  to break
>                  that trend.
>
>                  I've recently expanded my build to include the genomic and
>                  protein fasta
>                  sources for all the species in wormbase along the lines
>         of 48 fasta
>                  based sources .
>
>                  Loading them, I get this new stack trace.
>
>                  It seems that the mine might be opening a new database
>                  connection for
>                  each source.  Can anybody comment on the root cause &
>         how to fix it?
>
>                  Thanks!
>
>                  BUILD FAILED
>
>         /nfs/wormbase/wormmine/____website-intermine-dev/acedb-____dev/intermine/imbuild/____integrate.xml:54:
>
>                  The following error occurred while executing this line:
>
>         /nfs/wormbase/wormmine/____website-intermine-dev/acedb-____dev/intermine/bio/sources/____fasta/build.xml:46:
>                  org.intermine.objectstore.____ObjectStoreException:
>         Failed to
>                  instantiate
>                  IntegrationWriter class:
>
>         org.intermine.dataloader.____IntegrationWriterDataTrackingI____mpl
>                            at
>
>         org.intermine.dataloader.____IntegrationWriterFactory.____getIntegrationWriter(____IntegrationWriterFactory.java:____75)
>                            at
>
>         org.intermine.task.____DirectDataLoaderTask.____getIntegrationWriter(____DirectDataLoaderTask.java:85)
>                            at
>
>         org.intermine.task.____DirectDataLoaderTask.execute(____DirectDataLoaderTask.java:__129)
>                            at
>
>         org.intermine.bio.____dataconversion.____FastaLoaderTask.execute(____FastaLoaderTask.java:200)
>                            at
>
>         org.apache.tools.ant.____UnknownElement.execute(____UnknownElement.java:291)
>                            at
>
>         sun.reflect.____GeneratedMethodAccessor360.____invoke(Unknown
>         Source)
>                            at
>
>         sun.reflect.____DelegatingMethodAccessorImpl.____invoke(____DelegatingMethodAccessorImpl.____java:25)
>                            at
>         java.lang.reflect.Method.____invoke(Method.java:597)
>                            at
>
>         org.apache.tools.ant.dispatch.____DispatchUtils.execute(____DispatchUtils.java:106)
>                            at
>         org.apache.tools.ant.Task.____perform(Task.java:348)
>                            at
>         org.apache.tools.ant.Target.____execute(Target.java:390)
>                            at
>
>         org.apache.tools.ant.Target.____performTasks(Target.java:411)
>                            at
>
>         org.apache.tools.ant.Project.____executeSortedTargets(Project.____java:1360)
>                            at
>
>         org.apache.tools.ant.helper.____SingleCheckExecutor.____executeTargets(____SingleCheckExecutor.java:38)
>                            at
>
>         org.apache.tools.ant.Project.____executeTargets(Project.java:____1212)
>                            at
>                  org.apache.tools.ant.taskdefs.____Ant.execute(Ant.java:441)
>                            at
>
>         org.intermine.task.Integrate.____performAction(Integrate.java:____223)
>                            at
>
>         org.intermine.task.Integrate.____performAction(Integrate.java:____136)
>                            at
>
>         org.intermine.task.Integrate.____execute(Integrate.java:127)
>                            at
>
>         org.apache.tools.ant.____UnknownElement.execute(____UnknownElement.java:291)
>                            at
>
>         sun.reflect.____GeneratedMethodAccessor4.____invoke(Unknown Source)
>                            at
>
>         sun.reflect.____DelegatingMethodAccessorImpl.____invoke(____DelegatingMethodAccessorImpl.____java:25)
>                            at
>         java.lang.reflect.Method.____invoke(Method.java:597)
>                            at
>
>         org.apache.tools.ant.dispatch.____DispatchUtils.execute(____DispatchUtils.java:106)
>                            at
>         org.apache.tools.ant.Task.____perform(Task.java:348)
>                            at
>         org.apache.tools.ant.Target.____execute(Target.java:390)
>                            at
>
>         org.apache.tools.ant.Target.____performTasks(Target.java:411)
>                            at
>
>         org.apache.tools.ant.Project.____executeSortedTargets(Project.____java:1360)
>                            at
>
>         org.apache.tools.ant.Project.____executeTarget(Project.java:____1329)
>                            at
>
>         org.apache.tools.ant.helper.____DefaultExecutor.____executeTargets(____DefaultExecutor.java:41)
>                            at
>
>         org.apache.tools.ant.Project.____executeTargets(Project.java:____1212)
>                            at
>         org.apache.tools.ant.Main.____runBuild(Main.java:801)
>                            at
>         org.apache.tools.ant.Main.____startAnt(Main.java:218)
>                            at
>
>         org.apache.tools.ant.launch.____Launcher.run(Launcher.java:____280)
>                            at
>
>         org.apache.tools.ant.launch.____Launcher.main(Launcher.java:____109)
>                  Caused by: java.lang.reflect.____InvocationTargetException
>                            at
>
>         sun.reflect.____NativeMethodAccessorImpl.____invoke0(Native Method)
>                            at
>
>         sun.reflect.____NativeMethodAccessorImpl.____invoke(____NativeMethodAccessorImpl.java:____39)
>                            at
>
>         sun.reflect.____DelegatingMethodAccessorImpl.____invoke(____DelegatingMethodAccessorImpl.____java:25)
>                            at
>         java.lang.reflect.Method.____invoke(Method.java:597)
>                            at
>
>         org.intermine.dataloader.____IntegrationWriterFactory.____getIntegrationWriter(____IntegrationWriterFactory.java:____65)
>                            ... 34 more
>                  Caused by:
>         org.intermine.objectstore.____ObjectStoreException:
>
>                  Failed to
>                  instantiate ObjectStoreWriter class:
>
>         org.intermine.objectstore.____intermine.____ObjectStoreWriterInterMineImpl____,
>
>                  osAlias: os.production - java.lang.RuntimeException:
>         can't open
>                  datasource for {platform=PostgreSQL,
>                  datasource.dataSourceName=db.____production,
>                  datasource.maxConnections=50,
>                  datasource.databaseName=____wormmine-test-t,
>
>         datasource.class=org.____postgresql.ds.____PGPoolingDataSource,
>                  datasource.password=#####, datasource.user=intermine,
>                  driver=org.postgresql.Driver,
>         datasource.serverName=____localhost}
>                  org.postgresql.Driver PostgreSQL
>                            at
>
>         org.intermine.objectstore.____ObjectStoreWriterFactory.____getObjectStoreWriter(____ObjectStoreWriterFactory.java:____89)
>                            at
>
>         org.intermine.dataloader.____IntegrationWriterDataTrackingI____mpl.getInstance(____IntegrationWriterDataTrackingI____mpl.java:120)
>                            at
>
>         org.intermine.dataloader.____IntegrationWriterDataTrackingI____mpl.getInstance(____IntegrationWriterDataTrackingI____mpl.java:80)
>
>                            ... 39 more
>                  Caused by: java.lang.RuntimeException: can't open
>         datasource for
>                  {platform=PostgreSQL,
>         datasource.dataSourceName=db.____production,
>                  datasource.maxConnections=50,
>                  datasource.databaseName=____wormmine-test-t,
>
>         datasource.class=org.____postgresql.ds.____PGPoolingDataSource,
>                  datasource.password=#####, datasource.user=intermine,
>                  driver=org.postgresql.Driver,
>         datasource.serverName=____localhost}
>                  org.postgresql.Driver PostgreSQL
>                            at
>
>         org.intermine.sql.Database.____getConnection(Database.java:____106)
>                            at
>
>         org.intermine.objectstore.____intermine.____ObjectStoreInterMineImpl.____getConnection(____ObjectStoreInterMineImpl.java:____256)
>                            at
>
>         org.intermine.objectstore.____intermine.____ObjectStoreWriterInterMineImpl____.<init>(____ObjectStoreWriterInterMineImpl____.java:117)
>                            at
>
>         sun.reflect.____NativeConstructorAccessorImpl.____newInstance0(Native
>         Method)
>                            at
>
>         sun.reflect.____NativeConstructorAccessorImpl.____newInstance(____NativeConstructorAccessorImpl.____java:39)
>                            at
>
>         sun.reflect.____DelegatingConstructorAccessorI____mpl.newInstance(____DelegatingConstructorAccessorI____mpl.java:27)
>                            at
>
>         java.lang.reflect.Constructor.____newInstance(Constructor.__java:__513)
>                            at
>
>         org.intermine.objectstore.____ObjectStoreWriterFactory.____getObjectStoreWriter(____ObjectStoreWriterFactory.java:____78)
>                            ... 41 more
>                  Caused by: org.postgresql.util.____PSQLException: FATAL:
>
>                  connection limit
>                  exceeded for non-superusers
>                            at
>
>         org.postgresql.core.v3.____ConnectionFactoryImpl.____readStartupMessages(____ConnectionFactoryImpl.java:____464)
>                            at
>
>         org.postgresql.core.v3.____ConnectionFactoryImpl.____openConnectionImpl(____ConnectionFactoryImpl.java:____112)
>                            at
>
>         org.postgresql.core.____ConnectionFactory.____openConnection(____ConnectionFactory.java:66)
>                            at
>
>         org.postgresql.jdbc2.____AbstractJdbc2Connection.<init>____(AbstractJdbc2Connection.__java:__125)
>                            at
>
>         org.postgresql.jdbc3.____AbstractJdbc3Connection.<init>____(AbstractJdbc3Connection.__java:__30)
>                            at
>
>         org.postgresql.jdbc3g.____AbstractJdbc3gConnection.<____init>(____AbstractJdbc3gConnection.java:____22)
>                            at
>
>         org.postgresql.jdbc3g.____Jdbc3gConnection.<init>(____Jdbc3gConnection.java:24)
>                            at
>
>         org.postgresql.Driver.____makeConnection(Driver.java:____393)
>                            at
>         org.postgresql.Driver.connect(____Driver.java:267)
>                            at
>
>         java.sql.DriverManager.____getConnection(DriverManager.____java:582)
>                            at
>
>         java.sql.DriverManager.____getConnection(DriverManager.____java:185)
>                            at
>
>         org.postgresql.ds.common.____BaseDataSource.getConnection(____BaseDataSource.java:91)
>                            at
>
>         org.postgresql.ds.common.____BaseDataSource.getConnection(____BaseDataSource.java:74)
>                            at
>
>         org.postgresql.ds.____PGConnectionPoolDataSource.____getPooledConnection(____PGConnectionPoolDataSource.____java:60)
>                            at
>
>         org.postgresql.ds.jdbc23.____AbstractJdbc23PoolingDataSourc____e.getPooledConnection(____AbstractJdbc23PoolingDataSourc____e.java:429)
>                            at
>
>         org.postgresql.ds.jdbc23.____AbstractJdbc23PoolingDataSourc____e.getConnection(____AbstractJdbc23PoolingDataSourc____e.java:356)
>                            at
>
>         org.intermine.sql.Database.____getConnection(Database.java:____104)
>                            ... 48 more
>
>
>
>                  ___________________________________________________
>                  dev mailing list
>         [hidden email] <mailto:[hidden email]>
>         <mailto:[hidden email] <mailto:[hidden email]>>
>         http://mail.intermine.org/cgi-____bin/mailman/listinfo/dev
>         <http://mail.intermine.org/cgi-__bin/mailman/listinfo/dev>
>
>         <http://mail.intermine.org/__cgi-bin/mailman/listinfo/dev
>         <http://mail.intermine.org/cgi-bin/mailman/listinfo/dev>>
>
>
>

_______________________________________________
dev mailing list
[hidden email]
http://mail.intermine.org/cgi-bin/mailman/listinfo/dev