fairseq: Windows installation failure: fastBPE(setup.py) error

Ok. I have solved this problem myself. This error arises when you try to compile fastBPE( a dependency of fairseq) in windows. In order to solve this, you have to make modifications on:

• fastBPE source code
• python package distutils 's cygwinccompiler.py

First, git clone fastBPE source code to your local directory. Change line 15:

#include <mman.c>

to

#include "mman.c"

Add mman.h to fastBPE/fastBPE directory:

/*
 * sys/mman.h
 * mman-win32
 */

#ifndef _SYS_MMAN_H_
#define _SYS_MMAN_H_

#ifndef _WIN32_WINNT		// Allow use of features specific to Windows XP or later.                   
#define _WIN32_WINNT 0x0501	// Change this to the appropriate value to target other versions of Windows.
#endif						

/* All the headers include this file. */
#ifndef _MSC_VER
#include <_mingw.h>
#endif

#include <sys/types.h>

#ifdef __cplusplus
extern "C" {
#endif

#define PROT_NONE       0
#define PROT_READ       1
#define PROT_WRITE      2
#define PROT_EXEC       4

#define MAP_FILE        0
#define MAP_SHARED      1
#define MAP_PRIVATE     2
#define MAP_TYPE        0xf
#define MAP_FIXED       0x10
#define MAP_ANONYMOUS   0x20
#define MAP_ANON        MAP_ANONYMOUS

#define MAP_FAILED      ((void *)-1)

/* Flags for msync. */
#define MS_ASYNC        1
#define MS_SYNC         2
#define MS_INVALIDATE   4

void*   mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off);
int     munmap(void *addr, size_t len);
int     mprotect(void *addr, size_t len, int prot);
int     msync(void *addr, size_t len, int flags);
int     mlock(const void *addr, size_t len);
int     munlock(const void *addr, size_t len);

#ifdef __cplusplus
};
#endif

#endif /*  _SYS_MMAN_H_ */

and also mman.c

#include <windows.h>
#include <errno.h>
#include <io.h>

#include "mman.h"

#ifndef FILE_MAP_EXECUTE
#define FILE_MAP_EXECUTE    0x0020
#endif /* FILE_MAP_EXECUTE */

static int __map_mman_error(const DWORD err, const int deferr)
{
    if (err == 0)
        return 0;
    //TODO: implement
    return err;
}

static DWORD __map_mmap_prot_page(const int prot)
{
    DWORD protect = 0;
    
    if (prot == PROT_NONE)
        return protect;
        
    if ((prot & PROT_EXEC) != 0)
    {
        protect = ((prot & PROT_WRITE) != 0) ? 
                    PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ;
    }
    else
    {
        protect = ((prot & PROT_WRITE) != 0) ?
                    PAGE_READWRITE : PAGE_READONLY;
    }
    
    return protect;
}

static DWORD __map_mmap_prot_file(const int prot)
{
    DWORD desiredAccess = 0;
    
    if (prot == PROT_NONE)
        return desiredAccess;
        
    if ((prot & PROT_READ) != 0)
        desiredAccess |= FILE_MAP_READ;
    if ((prot & PROT_WRITE) != 0)
        desiredAccess |= FILE_MAP_WRITE;
    if ((prot & PROT_EXEC) != 0)
        desiredAccess |= FILE_MAP_EXECUTE;
    
    return desiredAccess;
}

void* mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off)
{
    HANDLE fm, h;
    
    void * map = MAP_FAILED;
    
#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable: 4293)
#endif

    const DWORD dwFileOffsetLow = (sizeof(off_t) <= sizeof(DWORD)) ? 
                    (DWORD)off : (DWORD)(off & 0xFFFFFFFFL);
    const DWORD dwFileOffsetHigh = (sizeof(off_t) <= sizeof(DWORD)) ?
                    (DWORD)0 : (DWORD)((off >> 32) & 0xFFFFFFFFL);
    const DWORD protect = __map_mmap_prot_page(prot);
    const DWORD desiredAccess = __map_mmap_prot_file(prot);

    const off_t maxSize = off + (off_t)len;

    const DWORD dwMaxSizeLow = (sizeof(off_t) <= sizeof(DWORD)) ? 
                    (DWORD)maxSize : (DWORD)(maxSize & 0xFFFFFFFFL);
    const DWORD dwMaxSizeHigh = (sizeof(off_t) <= sizeof(DWORD)) ?
                    (DWORD)0 : (DWORD)((maxSize >> 32) & 0xFFFFFFFFL);

#ifdef _MSC_VER
#pragma warning(pop)
#endif

    errno = 0;
    
    if (len == 0 
        /* Unsupported flag combinations */
        || (flags & MAP_FIXED) != 0
        /* Usupported protection combinations */
        || prot == PROT_EXEC)
    {
        errno = EINVAL;
        return MAP_FAILED;
    }
    
    h = ((flags & MAP_ANONYMOUS) == 0) ? 
                    (HANDLE)_get_osfhandle(fildes) : INVALID_HANDLE_VALUE;

    if ((flags & MAP_ANONYMOUS) == 0 && h == INVALID_HANDLE_VALUE)
    {
        errno = EBADF;
        return MAP_FAILED;
    }

    fm = CreateFileMapping(h, NULL, protect, dwMaxSizeHigh, dwMaxSizeLow, NULL);

    if (fm == NULL)
    {
        errno = __map_mman_error(GetLastError(), EPERM);
        return MAP_FAILED;
    }
  
    map = MapViewOfFile(fm, desiredAccess, dwFileOffsetHigh, dwFileOffsetLow, len);

    CloseHandle(fm);
  
    if (map == NULL)
    {
        errno = __map_mman_error(GetLastError(), EPERM);
        return MAP_FAILED;
    }

    return map;
}

int munmap(void *addr, size_t len)
{
    if (UnmapViewOfFile(addr))
        return 0;
        
    errno =  __map_mman_error(GetLastError(), EPERM);
    
    return -1;
}

int mprotect(void *addr, size_t len, int prot)
{
    DWORD newProtect = __map_mmap_prot_page(prot);
    DWORD oldProtect = 0;
    
    if (VirtualProtect(addr, len, newProtect, &oldProtect))
        return 0;
    
    errno =  __map_mman_error(GetLastError(), EPERM);
    
    return -1;
}

int msync(void *addr, size_t len, int flags)
{
    if (FlushViewOfFile(addr, len))
        return 0;
    
    errno =  __map_mman_error(GetLastError(), EPERM);
    
    return -1;
}

int mlock(const void *addr, size_t len)
{
    if (VirtualLock((LPVOID)addr, len))
        return 0;
        
    errno =  __map_mman_error(GetLastError(), EPERM);
    
    return -1;
}

int munlock(const void *addr, size_t len)
{
    if (VirtualUnlock((LPVOID)addr, len))
        return 0;
        
    errno =  __map_mman_error(GetLastError(), EPERM);
    
    return -1;
}

Reason : Under any windows C++ compiler, we don’t have mman.h . But I have found the following windows alternative lib.

@zjplab i followed your instructions 1.Downloaded fastBPE repo 2.added “if msc_ver >= ‘1900’: return [‘vcruntime140’]” in distutils\cygwinccompiler.py 3. created files mman.c,mman.h file with the code that you shared 4.replaced #include <sys/mman.h> with #include “mman.c” in fastBPE.hpp

Im getting this error ‘thread’ was not declared in this scope 28 | const size_t kThreads = max(1, min(10, int(thread::hardware_concurrency())))

Full stack trace: Tired to debug and search for this issue in other portals but was not able to get any solutions could you please assist

python setup.py build --compiler=mingw32 running build running build_py running build_ext building 'fastBPE' extension C:\MinGW\bin\gcc.exe -mdll -O -Wall -IfastBPE -IC:\Users\aa\AppData\Local\Programs\Python\Python37\include -IC:\Users\aa\AppData\Local\Programs\Python\Python37\include -c fastBPE/fastBPE.cpp -o build\temp.win-amd64-3.7\Release\fastbpe\fastbpe.o -std=c++11 -Ofast -pthread In file included from fastBPE/fastBPE.cpp:653: fastBPE/fastBPE.hpp:28:44: error: 'thread' was not declared in this scope 28 | const size_t kThreads = max(1, min(10, int(thread::hardware_concurrency()))); | ^~~~~~ fastBPE/fastBPE.hpp:20:1: note: 'std::thread' is defined in header '<thread>'; did you forget to '#include <thread>'? 19 | #include <unordered_set> +++ |+#include <thread> 20 | #include <vector> fastBPE/fastBPE.hpp: In function 'void fastBPE::readText(const char*, std::unordered_map<std::__cxx11::basic_string<char>, unsigned int>&)': fastBPE/fastBPE.hpp:83:27: warning: format '%lu' expects argument of type 'long unsigned int', but argument 3 has type 'uint64_t' {aka 'long long unsigned int'} [-Wformat=] 83 | fprintf(stderr, "Read %lu words (%lu unique) from text file.\n", total, | ~~^ ~~~~~ | | | | long unsigned int uint64_t {aka long long unsigned int} | %I64u fastBPE/fastBPE.hpp:83:38: warning: format '%lu' expects argument of type 'long unsigned int', but argument 4 has type 'std::unordered_map<std::__cxx11::basic_string<char>, unsigned int>::size_type' {aka 'unsigned int'} [-Wformat=] 83 | fprintf(stderr, "Read %lu words (%lu unique) from text file.\n", total, | ~~^ | | | long unsigned int | %u 84 | word_count.size()); | ~~~~~~~~~~~~~~~~~ | | | std::unordered_map<std::__cxx11::basic_string<char>, unsigned int>::size_type {aka unsigned int} fastBPE/fastBPE.hpp: In function 'void fastBPE::outputText(const char*, const char*, std::unordered_map<std::__cxx11::basic_string<char>, std::__cxx11::basic_string<char> >&)': fastBPE/fastBPE.hpp:136:7: error: 'ftruncate' was not declared in this scope; did you mean 'strncat'? 136 | if (ftruncate(fdOut, out_size) < 0) { | ^~~~~~~~~ | strncat fastBPE/fastBPE.hpp:137:65: warning: format '%lu' expects argument of type 'long unsigned int', but argument 4 has type 'size_t' {aka 'unsigned int'} [-Wformat=] 137 | fprintf(stderr, "Couldn't truncate output file %s to size %lu\n", fpo, | ~~^ | | | long unsigned int | %u 138 | out_size); | ~~~~~~~~ | | | size_t {aka unsigned int} fastBPE/fastBPE.hpp:149:31: warning: format '%lu' expects argument of type 'long unsigned int', but argument 3 has type 'long long unsigned int' [-Wformat=] 149 | fprintf(stderr, "Modified %lu words from text file.\n", p.second); | ~~^ ~~~~~~~~ | | | | long unsigned int long long unsigned int | %I64u fastBPE/fastBPE.hpp: In function 'void fastBPE::readVocab(const char*, std::unordered_map<std::__cxx11::basic_string<char>, unsigned int>&)': fastBPE/fastBPE.hpp:438:27: warning: format '%lu' expects argument of type 'long unsigned int', but argument 3 has type 'uint64_t' {aka 'long long unsigned int'} [-Wformat=] 438 | fprintf(stderr, "Read %lu words (%lu unique) from vocabulary file.\n", total, | ~~^ ~~~~~ | | | | long unsigned int uint64_t {aka long long unsigned int} | %I64u fastBPE/fastBPE.hpp:438:38: warning: format '%lu' expects argument of type 'long unsigned int', but argument 4 has type 'std::unordered_map<std::__cxx11::basic_string<char>, unsigned int>::size_type' {aka 'unsigned int'} [-Wformat=] 438 | fprintf(stderr, "Read %lu words (%lu unique) from vocabulary file.\n", total, | ~~^ | | | long unsigned int | %u 439 | vocab.size()); | ~~~~~~~~~~~~ | | | std::unordered_map<std::__cxx11::basic_string<char>, unsigned int>::size_type {aka unsigned int} fastBPE/fastBPE.hpp: In function 'void fastBPE::readCodes(const char*, std::unordered_map<std::pair<std::__cxx11::basic_string<char>, std::__cxx11::basic_string<char> >, unsigned int, fastBPE::pair_hash>&, std::unordered_map<std::__cxx11::basic_string<char>, std::pair<std::__cxx11::basic_string<char>, std::__cxx11::basic_string<char> > >&)': fastBPE/fastBPE.hpp:462:27: warning: format '%lu' expects argument of type 'long unsigned int', but argument 3 has type 'std::unordered_map<std::pair<std::__cxx11::basic_string<char>, std::__cxx11::basic_string<char> >, unsigned int, fastBPE::pair_hash>::size_type' {aka 'unsigned int'} [-Wformat=] 462 | fprintf(stderr, "Read %lu codes from the codes file.\n", codes.size()); | ~~^ ~~~~~~~~~~~~ | | | | long unsigned int std::unordered_map<std::pair<std::__cxx11::basic_string<char>, std::__cxx11::basic_string<char> >, unsigned int, fastBPE::pair_hash>::size_type {aka unsigned int} | %u fastBPE/fastBPE.hpp: In function 'void fastBPE::applybpe(const char*, const char*, const char*, const char*)': fastBPE/fastBPE.hpp:607:37: error: size of array 'bpe' is not an integral constant-expression 607 | unordered_map<string, string> bpe[kThreads]; | ^~~~~~~~ fastBPE/fastBPE.hpp:608:10: error: 'thread' was not declared in this scope 608 | vector<thread> threads; | ^~~~~~ fastBPE/fastBPE.hpp:608:10: note: 'std::thread' is defined in header '<thread>'; did you forget to '#include <thread>'? fastBPE/fastBPE.hpp:608:16: error: template argument 1 is invalid 608 | vector<thread> threads; | ^ fastBPE/fastBPE.hpp:608:16: error: template argument 2 is invalid fastBPE/fastBPE.hpp:610:13: error: request for member 'emplace_back' in 'threads', which is of non-class type 'int' 610 | threads.emplace_back( | ^~~~~~~~~~~~ fastBPE/fastBPE.hpp:623:14: error: invalid types 'int[size_t {aka unsigned int}]' for array subscript 623 | threads[i].join(); | ^ error: command 'C:\\MinGW\\bin\\gcc.exe' failed with exit status 1

where is “#include <mman.c>” ?

I have followed how to install fastBPE in Windows by @zjplab. However, when I run “python setup.py build --compiler=mingw32”, it produces an error: running build running build_py running build_ext building ‘fastBPE’ extension gcc -mdll -O -Wall -IfastBPE -IC:\Users\ASUS\anaconda3\envs\Testing_The_RS_AIP491\include -IC:\Users\ASUS\anaconda3\envs\Testing_The_RS_AIP491\Include -c fastBPE/fastBPE.cpp -o build\temp .win-amd64-cpython-38\Release\fastbpe\fastbpe.o -std=c++11 -Ofast -pthread In file included from fastBPE/fastBPE.cpp:789: fastBPE/fastBPE.hpp: In function ‘void fastBPE::readText(const char*, std::unordered_map<std::__cxx11::basic_string<char>, unsigned int>&)’: fastBPE/fastBPE.hpp:84:27: warning: format ‘%lu’ expects argument of type ‘long unsigned int’, but argument 3 has type ‘uint64_t’ {aka ‘long long unsigned int’} [-Wformat=] 84 | fprintf(stderr, “Read %lu words (%lu unique) from text file.\n”, total, | ~~^ ~~~~~ | | | | long unsigned int uint64_t {aka long long unsigned int} | %llu fastBPE/fastBPE.hpp:84:38: warning: format ‘%lu’ expects argument of type ‘long unsigned int’, but argument 4 has type ‘std::unordered_map<std::__cxx11::basic_string<char>, unsigned int>: :size_type’ {aka ‘unsigned int’} [-Wformat=] 84 | fprintf(stderr, “Read %lu words (%lu unique) from text file.\n”, total, | ~~^ | | | long unsigned int | %u 85 | word_count.size()); | ~~~~~~~~~~~~~~~~~ | | | std::unordered_map<std::__cxx11::basic_string<char>, unsigned int>::size_type {aka unsigned int} fastBPE/fastBPE.hpp: In function ‘void fastBPE::outputText(const char*, const char*, std::unordered_map<std::__cxx11::basic_string<char>, std::__cxx11::basic_string<char> >&)’: fastBPE/fastBPE.hpp:138:65: warning: format ‘%lu’ expects argument of type ‘long unsigned int’, but argument 4 has type ‘size_t’ {aka ‘unsigned int’} [-Wformat=] 138 | fprintf(stderr, “Couldn’t truncate output file %s to size %lu\n”, fpo, | ~~^ | | | long unsigned int | %u 139 | out_size); | ~~~~~~~~ | | | size_t {aka unsigned int} fastBPE/fastBPE.hpp:150:31: warning: format ‘%lu’ expects argument of type ‘long unsigned int’, but argument 3 has type ‘long long unsigned int’ [-Wformat=] 150 | fprintf(stderr, “Modified %lu words from text file.\n”, p.second); | ~~^ ~~~~~~~~ | | | | long unsigned int long long unsigned int | %llu fastBPE/fastBPE.hpp: In function ‘void fastBPE::readVocab(const char*, std::unordered_map<std::__cxx11::basic_string<char>, unsigned int>&)’: fastBPE/fastBPE.hpp:439:27: warning: format ‘%lu’ expects argument of type ‘long unsigned int’, but argument 3 has type ‘uint64_t’ {aka ‘long long unsigned int’} [-Wformat=] 439 | fprintf(stderr, “Read %lu words (%lu unique) from vocabulary file.\n”, total, | ~~^ ~~~~~ | | | | long unsigned int uint64_t {aka long long unsigned int} | %llu fastBPE/fastBPE.hpp:439:38: warning: format ‘%lu’ expects argument of type ‘long unsigned int’, but argument 4 has type ‘std::unordered_map<std::__cxx11::basic_string<char>, unsigned int> ::size_type’ {aka ‘unsigned int’} [-Wformat=] 439 | fprintf(stderr, “Read %lu words (%lu unique) from vocabulary file.\n”, total, | ~~^ | | | long unsigned int | %u 440 | vocab.size()); | ~~~~~~~~~~~~ | | | std::unordered_map<std::__cxx11::basic_string<char>, unsigned int>::size_type {aka unsigned int} fastBPE/fastBPE.hpp: In function ‘void fastBPE::readCodes(const char*, std::unordered_map<std::pair<std::__cxx11::basic_string<char>, std::__cxx11::basic_string<char> >, unsigned int, pai r_hash>&, std::unordered_map<std::__cxx11::basic_string<char>, std::pair<std::__cxx11::basic_string<char>, std::__cxx11::basic_string<char> > >&)’: fastBPE/fastBPE.hpp:463:27: warning: format ‘%lu’ expects argument of type ‘long unsigned int’, but argument 3 has type ‘std::unordered_map<std::pair<std::__cxx11::basic_string<char>, std ::__cxx11::basic_string<char> >, unsigned int, fastBPE::pair_hash>::size_type’ {aka ‘unsigned int’} [-Wformat=] 463 | fprintf(stderr, “Read %lu codes from the codes file.\n”, codes.size()); | ~~^ ~~~~~~~~~~~~ | | | | long unsigned int std::unordered_map<std::pair<std::__cxx11::basic_string<char>, std::__cxx11::basic_string<char> >, unsigned in t, fastBPE::pair_hash>::size_type {aka unsigned int} | %u writing build\temp.win-amd64-cpython-38\Release\fastbpe\fastBPE.cp38-win_amd64.def g++ -O -Wall -shared -s build\temp.win-amd64-cpython-38\Release\fastbpe\fastbpe.o build\temp.win-amd64-cpython-38\Release\fastbpe\fastBPE.cp38-win_amd64.def -LC:\Users\ASUS\anaconda3\envs \Testing_The_RS_AIP491\libs -LC:\Users\ASUS\anaconda3\envs\Testing_The_RS_AIP491 -LC:\Users\ASUS\anaconda3\envs\Testing_The_RS_AIP491\PCbuild\amd64 -lpython38 -lucrt -lvcruntime140 -o bui ld\lib.win-amd64-cpython-38\fastBPE.cp38-win_amd64.pyd C:/mingw32/bin/…/lib/gcc/i686-w64-mingw32/12.2.0/…/…/…/…/i686-w64-mingw32/bin/ld.exe: C:\Users\ASUS\anaconda3\envs\Testing_The_RS_AIP491\libs/python38.lib(python38.dll): recognised b ut unhandled machine type (0x8664) in Import Library Format archive C:/mingw32/bin/…/lib/gcc/i686-w64-mingw32/12.2.0/…/…/…/…/i686-w64-mingw32/bin/ld.exe: C:\Users\ASUS\anaconda3\envs\Testing_The_RS_AIP491\libs/python38.lib: error adding symbols: file format not recognized collect2.exe: error: ld returned 1 exit status error: command ‘C:\mingw32\bin\g++.exe’ failed with exit code 1

How can I fix it ?

Just adding comments of own experiences to help others with this. For reference I’m trying to get fastBPE for use for the salesforce CTRL model here: https://github.com/salesforce/ctrl

Problem 1:

• Error 1: I followed @zjplab instructions which worked from step 1 but came up with errors similar to @ajbot2019 for step 2.
• In order to fix, I had to download a specific 64 bit version of mingw. Previous versions don’t have support for c++ threads.

Problem 2:

• Error 1: After doing this I had an error message for lvcruntime140 with “file type not recognized”.
• In order to fix I had to use a specific 64 bit version of vcruntime140.dll and placed it into the same file as step 3 of @zjplab instructions.
• @ajbot2019 these steps should fix your errors hopefully.

Problem 3:

• Error 1: A number of error messages along the lines of "undefined reference to ‘_int___FUNCTIONNAMEHERE’.
• Error 2: message stating "bad reloc address 0x8 in section ‘.data’ NEWLINE ‘collect2.exe: error 1d returned 1 exit status’.
• Currently trying to fix. Will update with results shortly.

UPDATE:

• Problem 3, Error 1: These are caused by the functions not being findable by the gcc linker (ld.exe). This website gives more detail: https://latedev.wordpress.com/2014/04/22/common-c-error-messages-2-unresolved-reference/#:~:text=Undefined reference Put simply%2C the “undefined reference” error,files and libraries that make up your project.
• Attempted to force the compiler to use G++ as outlined by the fastBPE spec at: https://pypi.org/project/fastBPE/ same issues occurred.
• All commands with “undefined reference” error are part of the libpython37.a static library which the LC.exe linker is not referencing.
• Attempting to work around.

UPDATE 2:

• I’ve given up and run this on a Linux dualboot which is causing no issues. If I discover a windows fix then I’ll post here but for the meantime don’t expect anything. Best of luck!