Matlab Engine - passing strings giving invalid characters

Hi,

I am writing an Engine application in C, which will be standalone eventually, but I would also like to call the routines via a mex interface if possible. So, at the moment I have a rather convoluted process of a mex file creating a separate instance of the Matlab engine, which I want to talk to (eventually this will be multiple engines via MPI). I'm doing this using Matlab coder.

So, I have the test code below which is trying to open a new engine from a mex file and talk to it...

function outp = engTest()

%Build with....    codegen engTest -o engTest_mex

params = [1, 2, 3, 4, 5, 6, 7, 8];
paramsLen = 8;
funName = 'total = debugMfile(params,bulk_in,bulk_out,contrast);';
pathCall = 'cd(''/home/arwel/Documents/coding/cevalTests/mlEng'');';
bulkIn = 2.073e-6;
bulkOut = 6.35e-6;
contrast = 1.0;

path = '/home/arwel/Documents/coding/cevalTests/mlEng';

incPath1 = '/usr/local/MATLAB/R2015b/extern/include';
incPath2 = '/usr/include/openmpi';

linkPath1 = '/usr/local/MATLAB/R2015b/bin/glnxa64';
linkFile1 = 'libeng.so';
linkFile2 = 'libmx.so';

source1 = 'matlabCallFun.c';
source2 = 'matlabEngine_demo.h';

libPriority = '';
libPreCompiled = true;
libLinkOnly = true;
%libName = 'LinkObj.lib';
%libPath = 'c:\Link_Objects';
%coder.updateBuildInfo('addLinkObjects', libName, libPath, ...
     %libPriority, libPreCompiled, libLinkOnly);

%coder.updateBuildInfo('addSourceFiles',filename)
coder.cinclude(source2);
coder.updateBuildInfo('addSourceFiles',source1);
coder.updateBuildInfo('addSourcePaths',path);
coder.updateBuildInfo('addIncludePaths',incPath1);
coder.updateBuildInfo('addIncludePaths',incPath2);
coder.updateBuildInfo('addLinkObjects',linkFile1,linkPath1,libPriority,libPreCompiled,libLinkOnly);
coder.updateBuildInfo('addLinkObjects',linkFile2,linkPath1,libPriority,libPreCompiled,libLinkOnly);

outp = zeros(2,3);

%matlabCallFun(params, paramsLen, funName, pathCall, bulkIn, bulkOut, contrast, s);
coder.ceval('matlabCallFun', params, paramsLen, funName, pathCall, bulkIn, bulkOut, contrast, coder.wref(outp)); 

end

and matlabCallFun.c as follows.....

    /*
     * matlabCallFun.c
     *
     *  Created on: 21 Jul 2017
     *      Author: arwel
     */
    #include <stdlib.h>
    #include <stdio.h>
    #include <string.h>

    #include "engine.h"

    void matlabCallFun(double params[], int arrayLen, char *funName, char *pathCall, double bulkIn, double bulkOut, double contrast, double *sum) {
      static Engine *ep;
      static double engStatus = 0;
      mxArray *result = NULL;

      mxArray *PARAMS = NULL;
      mxArray *BULKIN = NULL;
      mxArray *BULKOUT = NULL;
      mxArray *CONTRAST = NULL;
        mxArray *FNAME = mxCreateString(funName);
        double *s;

      if(engStatus == 0) {
        ep = engOpen("");    
        if(ep==0) {
          printf("Connecton to Matlab Engine failed\n");
        }
        else {
          printf("Connecton to Matlab Engine succeeded!\n");
          engEvalString(ep,(void *)pathCall);
          engStatus = 1;
        }
      }

      PARAMS = mxCreateDoubleMatrix(1,arrayLen,mxREAL);
      memcpy((void *)mxGetPr(PARAMS), (void *)params, arrayLen*sizeof(double));
      engPutVariable(ep,"params",PARAMS);

      BULKIN = mxCreateDoubleMatrix(1,1,mxREAL);
      memcpy((void *)mxGetPr(BULKIN), &bulkIn, 1*sizeof(double));
      engPutVariable(ep,"bulk_in",BULKIN);

      BULKOUT = mxCreateDoubleMatrix(1,1,mxREAL);
      memcpy((void *)mxGetPr(BULKOUT), &bulkOut, 1*sizeof(double));
      engPutVariable(ep,"bulk_out",BULKOUT);

      CONTRAST = mxCreateDoubleMatrix(1,1,mxREAL);
      memcpy((void *)mxGetPr(CONTRAST), &contrast, 1*sizeof(double));
      engPutVariable(ep,"contrast",BULKOUT);

      engEvalString(ep,(void *)funName);

      result = engGetVariable(ep,"total");
      s = (double *)mxGetData(result);

        memcpy(sum, s, 8*sizeof(double));

      engClose(ep);

}

When I compile this, and try to run

codegen engTest -o engTest_mex
y = engTest_mex

I get a seg fault, and the following in the relevant terminal window....

Connecton to Matlab Engine succeeded!
 cd('/home/arwel/Documents/coding/cevalTests/mlEng');7
                                                      |
Error: The input character is not valid in MATLAB statements or expressions.

   total = debugMfile(params,bulk_in,bulk_out,contrast);,YǂZ
                                                          |
  Error: The input character is not valid in MATLAB statements or expressions.

What am I missing here?

Cheers,

Arwel

p.s.

The fun being called is....

function total = debugMfile(params,bulk_in,bulk_out,contrast)

s = sum(params);
total = [s bulk_out bulk_in ; 4 5 6];
save debugMfileVars.mat

end

[Arwel]

p.p.s. I should also point out that matlabCallFun runs just fine when called from a 'main' like this....

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
//#include <mpi.h>
#include "engine.h"

void matlabCallFun(double params[], int arrayLen, char *funName, char *pathCall, double bulkIn, double bulkOut, double contrast, double *sum);

int main(int argc, char** argv) {

  double params[] = {1, 2, 3, 4, 5, 6, 7, 8};
  int paramsLen = 8;
  char funName[60] = "total = debugMfile(params,bulk_in,bulk_out,contrast)";
  char pathCall[70] = "cd('/home/arwel/Documents/MATLAB/mNestFiles');";
  double bulkIn = 2.073e-6;
  double bulkOut = 6.35e-6;
  double contrast = 1.0;
  double *s;
  //int bulkOut = 6.35e-6;
  //int contrast = 1;

  //Initialise MPI
  //MPI_Init(NULL, NULL);

  //[layersList,sRoughs] = custom_layers(params,whichContrast,fileHandle,nbairs,nbsubs);

  matlabCallFun(params, paramsLen, funName, pathCall, bulkIn, bulkOut, contrast, s);

  return(0);

}

Answer 1

[James Tursa]

I don't see how this can possibly work, even when called from main( ). It appears you are writing into invalid memory. E.g.,

outp = zeros(2,3);
coder.ceval('matlabCallFun', params, paramsLen, funName, pathCall, bulkIn, bulkOut, contrast, coder.wref(outp));

In the above lines, it appears you have pre-allocated a total of 6 double elements for "outp". Yet in your C code there is this:

    void matlabCallFun(double params[], int arrayLen, char *funName, char *pathCall, double bulkIn, double bulkOut, double contrast, double *sum) {
        :
        memcpy(sum, s, 8*sizeof(double));

So it appears in your C code you are writing 8 double elements. Seems like this would run off the end of valid memory.

And in your main( ) example you have this:

int main(int argc, char** argv) {
    :
  double *s;
          :
  matlabCallFun(params, paramsLen, funName, pathCall, bulkIn, bulkOut, contrast, s);

So here it appears you are passing an un-initialized pointer into matlabCallFun, whereupon matlabCallFun attempts to write 8 doubles to this invalid memory location.

Bottom line is I don't see how this could work in either case.

[Arwel]

Hi James, The 8 doubles versus 6 is indeed a typo. In terms of the pointer, this is how to get results out from a coder.ceval code i.e. with reference.

I actually contacted TMW support to ask about this, and here is the example they sent me....

foo.c

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "foo.h"

void foo(double *in1)
{
  double params[] = {1,2,3,4,5,6,7,8}; 
  memcpy(in1, params, 8*sizeof(double)); 

}

foo.h

void foo(double *in1) ;

test.m

function y= test

y = zeros (1, 8); 

coder.ceval('foo', coder.wref(y)); 

end

Then....

>>  codegen test foo.c foo.h -o test_mex
>>  y= test_mex

y =

       1     2     3     4     5     6     7     8

[James Tursa]

This latest posted example answers the 8 vs 6 issue. But it does not address the uninitialized pointer issue that I mention above in your main( ) example. In that case the pointer "s" is declared but given no value ... it is uninitialized. It is then passed into your matlabCallFun routine and used as the target for a memcpy. So you are writing to an invalid address, and anything bad can happen at that point.

Answer 2

[Arwel]

Thanks James. I really appreciate you looking at this. Parking the 'invalid characters' problem for now by hard coding in the paths etc, the following now runs at least....

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "engine.h"

void matlabCallFun(double params[], int arrayLen, char *funName, char *pathCall, double bulkIn, double bulkOut, double contrast, double *sum)
{
  static Engine *ep;
  static double engStatus = 0;
  mxArray *result = NULL;
  mxArray *PARAMS = NULL;
  mxArray *BULKIN = NULL;
  mxArray *BULKOUT = NULL;
  mxArray *CONTRAST = NULL;
  double* s;

  double p[6] = {1,2,3,4,5,6};

  if(engStatus == 0) {
    ep = engOpen("");    
    if(ep==0) {
      printf("Connecton to Matlab Engine failed\n");
    }
    else {
      printf("Connecton to Matlab Engine succeeded!\n");
      /*engEvalString(ep,(char *)pathCall);*/
      engEvalString(ep,"cd('/home/arwel/eclipseWorkspace_new/matlabEngine_demo/src')");
      engStatus = 1;
    }
  }

  PARAMS = mxCreateDoubleMatrix(1,arrayLen,mxREAL);
  memcpy((void *)mxGetPr(PARAMS), (void *)params, arrayLen*sizeof(double));
  engPutVariable(ep,"params",PARAMS);

  BULKIN = mxCreateDoubleMatrix(1,1,mxREAL);
  memcpy((void *)mxGetPr(BULKIN), &bulkIn, 1*sizeof(double));
  engPutVariable(ep,"bulk_in",BULKIN);

  BULKOUT = mxCreateDoubleMatrix(1,1,mxREAL);
  memcpy((void *)mxGetPr(BULKOUT), &bulkOut, 1*sizeof(double));
  engPutVariable(ep,"bulk_out",BULKOUT);

  CONTRAST = mxCreateDoubleMatrix(1,1,mxREAL);
  memcpy((void *)mxGetPr(CONTRAST), &contrast, 1*sizeof(double));
  engPutVariable(ep,"contrast",BULKOUT);

  engEvalString(ep,"total = debugMfile(params,bulk_in,bulk_out,contrast)");

  result = engGetVariable(ep,"total");
  s = (double *)mxGetData(result);
  memcpy(sum,s,8*sizeof(double));

  printf("In function, s[0] is %f \n",sum[0]);

  engClose(ep);

};

.... and from Matlab....

function outp = engTest()

%Build with....    codegen engTest -o engTest_mex

params = [1, 2, 3, 4, 5, 6, 7, 8];
paramsLen = 8;
funName = 'total = debugMfile(params,bulk_in,bulk_out,contrast);';
pathCall = 'cd(''/home/arwel/eclipseWorkspace_new/matlabEngine_demo/src'');';
bulkIn = 2.073e-6;
bulkOut = 6.35e-6;
contrast = 1.0;

path = '/home/arwel/eclipseWorkspace_new/matlabEngine_demo/src';

incPath1 = '/usr/local/MATLAB/R2015b/extern/include';
incPath2 = '/usr/include/openmpi';

linkPath1 = '/usr/local/MATLAB/R2015b/bin/glnxa64';
linkFile1 = 'libeng.so';
linkFile2 = 'libmx.so';

source1 = 'matlabCallFun.c';
source2 = 'matlabCallFun.h';

libPriority = '';
libPreCompiled = true;
libLinkOnly = true;
%libName = 'LinkObj.lib';
%libPath = 'c:\Link_Objects';
%coder.updateBuildInfo('addLinkObjects', libName, libPath, ...
     %libPriority, libPreCompiled, libLinkOnly);

%coder.updateBuildInfo('addSourceFiles',filename)
coder.cinclude(source2);
coder.updateBuildInfo('addSourceFiles',source1);
coder.updateBuildInfo('addSourcePaths',path);
coder.updateBuildInfo('addIncludePaths',incPath1);
coder.updateBuildInfo('addIncludePaths',incPath2);
coder.updateBuildInfo('addLinkObjects',linkFile1,linkPath1,libPriority,libPreCompiled,libLinkOnly);
coder.updateBuildInfo('addLinkObjects',linkFile2,linkPath1,libPriority,libPreCompiled,libLinkOnly);

outp = zeros(10,3);

%matlabCallFun(params, paramsLen, funName, pathCall, bulkIn, bulkOut, contrast, s);
coder.ceval('matlabCallFun', params, paramsLen, funName, pathCall, bulkIn, bulkOut, contrast, coder.wref(outp)); 

end

...calling the following m-file....

function total = debugMfile(params,bulk_in,bulk_out,contrast)

s = sum(params);
total = [s 37 38; 4 5 6];
save debugMfileVars.mat

end

Then.....

>> codegen engTest -o engTest_mex
>> y = engTest_mex

y =

     36.0000    0.0000    0.0000
      4.0000         0    0.0000
     37.0000    0.0000    0.0000
      5.0000    0.0000    0.0000
     38.0000    0.0000    0.0000
      6.0000    0.0000    0.0000
           0    0.0000    0.0000
      0.0000    0.0000    0.0000
      0.0000    0.0000    0.0000
      0.0000    0.0000         0

So...

(a) If I run it twice, then it seg faults... I think because of the lack of mxDestroyArray maybe?

(b) Everything is getting back to matlab, but screwed up in terms of ordering. We're expecting [35 36 37;4 5 6]

and (c) I still need to be able to pass the function name as a parameter, rather than hard coding it..

Closer, but still no cigar..

Cheers,

Arwel

[Arwel]

(oh...... and the final array produced by the m-file is always going to be m x 3, with m unknown..... that too..)

[James Tursa]

Some comments:

1) The prototype for memcpy has (void *) for the pointer arguments. You get conversions from any object pointer to (void *) automatically, so there is no need to explicitly cast these pointers in your memcpy calls. E.g., this line

memcpy((void *)mxGetPr(PARAMS), (void *)params, arrayLen*sizeof(double));

can be this line instead, which maybe is more readable (at least to me anyway):

memcpy( mxGetPr(PARAMS), params, arrayLen*sizeof(double) );

2) There is a simpler way to create double scalar mxArray variables than what you are doing. E.g. these lines:

BULKIN = mxCreateDoubleMatrix(1,1,mxREAL);
memcpy((void *)mxGetPr(BULKIN), &bulkIn, 1*sizeof(double));

can be replaced with this line:

BULKIN = mxCreateDoubleScalar(bulkIn);

3) The ordering of the values appears to be exactly as expected to me. You create the variable "outp" as a 10x3 double and then pass the address of that into matlabCallFun via "coder.wref(outp)". Calculations are then done in the Engine with a call to debugMfile where the return variable is a 2D matrix total = [36 37 38; 4 5 6]. But MATLAB stores variable data in column ordered fashion. So these six values are actually stored in memory in the following order: 36, 4, 37, 5, 38, 6. Then back in the matlabCallFun code you copy these values into "outp" using a memcpy call ... which will simply retain the ordering in memory. Bottom line is I would expect these values to show up in the first column of "outp" in the same order that they are in memory in the "total" variable, and that is exactly what I am seeing. So this is all as expected and no surprise. If you want something else to show up you will need to code the value copying differently. I could advise you here, but I don't really know what you want. Is it your intention to have "outp" pre-allocated with more rows than necessary, and then get filled with "total" by row, so that the top part of "outp" will be a copy of "total"? Or what? If it is something like this, then you will need to pass in the size of "outp" to matlabCallFun so that your C code knows how to copy the values from "total" into the appropriate spots of "outp". Let me know if you need help with this.

4) You still have a coding error with writing to invalid memory. This seems to be a persistent problem with your code. In particular, note that "total" has only 6 elements, but you are still using that hardcoded 8 element memcpy. So you are reading off the end of the valid data block of memory. This is likely what is causing the seg fault. To fix this, code your value copying more robustly:

memcpy( sum, s, mxGetNumberOfElements(result)*sizeof(double) );

But this only solves half the problem, since the "sum" target of the copy might not be large enough to hold what is in "result". So you will probably want to adjust your code to ensure this doesn't happen. Either something to ensure "outp" is large enough to hold "total" in your m-code, or maybe pass in some sizing information to matlabCallFun so that this can be checked there and guard against invalid memory copies.

5) Not calling mxDestroyArray in and of itself will not result in a seg fault. The issue will be one of leaking memory and eventually running out of memory depending on the size of the mxArray variables involved. But, even though this is not the cause of the seg fault, you should get in the habit of destroying those mxArrays you create when you no longer need them in your code to avoid those memory leaks. In a mex routine, the memory leaks will be temporary and will be garbage collected when the mex routine returns to the caller. But in a true standalone Engine app, the memory leaks will be permanent until the program exits.

6) I am puzzled why some of your output is 0 and some is 0.0000. Typically the latter indicates that the value is not exactly 0. It is puzzling to me because "outp" was initialized to exactly 0's and I see nothing in your code that would change those trailing values to something else.

Answer 3

[Arwel]

Hi James,

Okay.... some developments....

I made some of the changes you suggested, except for mxDoubleScalar since BulkIn is a floating point double, so that it now looks like this....

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "engine.h"

void matlabCallFun(double params[], int arrayLen, char *funName, char *pathCall, double bulkIn, double bulkOut, double contrast, double *sum)
{
  static Engine *ep;
  static double engStatus = 0;
  mxArray *result = NULL;
  mxArray *PARAMS = NULL;
  mxArray *BULKIN = NULL;
  mxArray *BULKOUT = NULL;
  mxArray *CONTRAST = NULL;
  double* s;

  double p[6] = {1,2,3,4,5,6};

  if(engStatus == 0) {
    ep = engOpen("");    
    if(ep==0) {
      printf("Connecton to Matlab Engine failed\n");
    }
    else {
      printf("Connecton to Matlab Engine succeeded!\n");
      /*engEvalString(ep,(char *)pathCall);*/
      engEvalString(ep,"cd('/home/arwel/eclipseWorkspace_new/matlabEngine_demo/src')");
      engStatus = 1;
    }
  }

  PARAMS = mxCreateDoubleMatrix(1,arrayLen,mxREAL);
  memcpy(mxGetPr(PARAMS), params, arrayLen*sizeof(double));
  engPutVariable(ep,"params",PARAMS);

  BULKIN = mxCreateDoubleMatrix(1,1,mxREAL);
  memcpy((void *)mxGetPr(BULKIN), &bulkIn, 1*sizeof(double));
  engPutVariable(ep,"bulk_in",BULKIN);

  BULKOUT = mxCreateDoubleMatrix(1,1,mxREAL);
  memcpy((void *)mxGetPr(BULKOUT), &bulkOut, 1*sizeof(double));
  engPutVariable(ep,"bulk_out",BULKOUT);

  CONTRAST = mxCreateDoubleMatrix(1,1,mxREAL);
  memcpy((void *)mxGetPr(CONTRAST), &contrast, 1*sizeof(double));
  engPutVariable(ep,"contrast",BULKOUT);

  engEvalString(ep,"total = debugMfile(params,bulk_in,bulk_out,contrast)");

  result = engGetVariable(ep,"total");
  s = (double *)mxGetData(result);
  memcpy( sum, s, mxGetNumberOfElements(result)*sizeof(double) );

  printf("In function, s[0] is %f \n",sum[0]);

  /*engClose(ep)*/;

  mxDestroyArray(PARAMS);
  mxDestroyArray(BULKIN);
  mxDestroyArray(BULKOUT);
  mxDestroyArray(CONTRAST);

};

Now I can run as often as needed with no segfaulting....

>> clear all
>> codegen engTest -o engTest_mex
>> tic;y = engTest_mex;toc;y
Elapsed time is 9.153074 seconds.

y =

     36.0000    0.0000    0.0000
      4.0000    0.0000    0.0000
     37.0000    0.0000    0.0000
      5.0000    0.0000    0.0000
     38.0000    0.0000    0.0000
      6.0000    0.0000    0.0000
      0.0000    0.0000    0.0000
      0.0000    0.0000         0
      0.0000    0.0000    0.0000
      0.0000    0.0000    0.0000

>> tic;y = engTest_mex;toc;y
Elapsed time is 0.037946 seconds.

y =

     36.0000    0.0000    0.0000
      4.0000    0.0000    0.0000
     37.0000    0.0000    0.0000
      5.0000    0.0000    0.0000
     38.0000    0.0000    0.0000
      6.0000    0.0000    0.0000
      0.0000    0.0000    0.0000
      0.0000    0.0000    0.0000
      0.0000    0.0000    0.0000
      0.0000    0.0000    0.0000

>>

There are still those 0 or 0.0000 which turn up in different places, but I'm not overly worried, since...

>> y(1,2)

ans =

    1.2461e-306

>>

..and...

>> tic;y = engTest_mex;toc;y;
Elapsed time is 0.023215 seconds.
>> yy = y(1:5,1)

yy =

      36
       4
      37
       5
      38

>> tic;y = engTest_mex;toc;y;
Elapsed time is 0.033529 seconds.
>> yy2 = y(1:5,1)

yy2 =

      36
       4
      37
       5
      38

>> isequal(yy,yy2)

ans =

    logical

     1

>>

" Is it your intention to have "outp" pre-allocated with more rows than necessary, and then get filled with "total" by row, so that the top part of "outp" will be a copy of "total"? Or what? If it is something like this, then you will need to pass in the size of "outp" to matlabCallFun so that your C code knows how to copy the values from "total" into the appropriate spots of "outp". Let me know if you need help with this."

Yes, basically, that is what I'm trying to do. I would welcome your advice on that. However, in principle the user script could make result any m x 3, so I guess allocating a large block and throwing an error if the script tries to make m too large is all I can do. That feels a little clumsy however, and some form of dynamic allocation based on the size of m would be better, but m is only known at runtime, so I guess not feasible in C. But, having a dynamically sized size m x 3 array would be the holy grail here.

That the leaves the original problem of...

     total = debugMfile(params,bulk_in,bulk_out,contrast);,YǂZ
                                                            |
    Error: The input character is not valid in MATLAB statements or expressions.

..if I want to use the script name passed at runtime. I'm guessing there will be some 'mx' command in the API to convert between C and Matlab strings?